27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/PostOrderIterator.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/STLForwardCompat.h"
31#include "llvm/ADT/SmallString.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/ADT/TypeSwitch.h"
34#include "llvm/ADT/bit.h"
35#include "llvm/Support/InterleavedRange.h"
41#include "mlir/Dialect/OpenMP/OpenMPOpsDialect.cpp.inc"
42#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.cpp.inc"
43#include "mlir/Dialect/OpenMP/OpenMPOpsInterfaces.cpp.inc"
44#include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.cpp.inc"
51 return attrs.empty() ?
nullptr : ArrayAttr::get(context, attrs);
65struct MemRefPointerLikeModel
66 :
public PointerLikeType::ExternalModel<MemRefPointerLikeModel,
69 return llvm::cast<MemRefType>(pointer).getElementType();
73struct LLVMPointerPointerLikeModel
74 :
public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,
75 LLVM::LLVMPointerType> {
100 bool isRegionArgOfOp;
110 assert(isRegionArgOfOp &&
"Must describe a region operand");
113 size_t &getArgIdx() {
114 assert(isRegionArgOfOp &&
"Must describe a region operand");
119 assert(!isRegionArgOfOp &&
"Must describe a operation of a region");
123 assert(!isRegionArgOfOp &&
"Must describe a operation of a region");
126 bool isLoopOp()
const {
127 assert(!isRegionArgOfOp &&
"Must describe a operation of a region");
128 return isa<CanonicalLoopOp>(op);
130 Region *&getParentRegion() {
131 assert(!isRegionArgOfOp &&
"Must describe a operation of a region");
134 size_t &getLoopDepth() {
135 assert(!isRegionArgOfOp &&
"Must describe a operation of a region");
139 void skipIf(
bool v =
true) { skip = skip || v; }
157 llvm::ReversePostOrderTraversal<Block *> traversal(&r->
getBlocks().front());
160 size_t sequentialIdx = -1;
161 bool isOnlyContainerOp =
true;
162 for (
Block *
b : traversal) {
164 if (&op == o && !found) {
168 if (op.getNumRegions()) {
171 isOnlyContainerOp =
false;
173 if (found && !isOnlyContainerOp)
178 Component &containerOpInRegion = components.emplace_back();
179 containerOpInRegion.isRegionArgOfOp =
false;
180 containerOpInRegion.isUnique = isOnlyContainerOp;
181 containerOpInRegion.getContainerOp() = o;
182 containerOpInRegion.getOpPos() = sequentialIdx;
183 containerOpInRegion.getParentRegion() = r;
188 Component ®ionArgOfOperation = components.emplace_back();
189 regionArgOfOperation.isRegionArgOfOp =
true;
190 regionArgOfOperation.isUnique =
true;
191 regionArgOfOperation.getArgIdx() = 0;
192 regionArgOfOperation.getOwnerOp() = parent;
204 for (
auto [idx, region] : llvm::enumerate(o->
getRegions())) {
208 llvm_unreachable(
"Region not child of its parent operation");
210 regionArgOfOperation.isUnique =
false;
211 regionArgOfOperation.getArgIdx() = getRegionIndex(parent, r);
219 for (Component &c : components)
220 c.skipIf(c.isRegionArgOfOp && c.isUnique);
223 size_t numSurroundingLoops = 0;
224 for (Component &c : llvm::reverse(components)) {
229 if (c.isRegionArgOfOp) {
230 numSurroundingLoops = 0;
237 numSurroundingLoops = 0;
239 c.getLoopDepth() = numSurroundingLoops;
242 if (isa<CanonicalLoopOp>(c.getContainerOp()))
243 numSurroundingLoops += 1;
248 bool isLoopNest =
false;
249 for (Component &c : components) {
250 if (c.skip || c.isRegionArgOfOp)
253 if (!isLoopNest && c.getLoopDepth() >= 1) {
256 }
else if (isLoopNest) {
258 c.skipIf(c.isUnique);
262 if (c.getLoopDepth() == 0)
269 for (Component &c : components)
270 c.skipIf(!c.isRegionArgOfOp && c.isUnique &&
271 !isa<CanonicalLoopOp>(c.getContainerOp()));
275 bool newRegion =
true;
276 for (Component &c : llvm::reverse(components)) {
277 c.skipIf(newRegion && c.isUnique);
284 if (!c.isRegionArgOfOp && c.getContainerOp())
290 llvm::raw_svector_ostream NameOS(Name);
291 for (
auto &c : llvm::reverse(components)) {
295 if (c.isRegionArgOfOp)
296 NameOS <<
"_r" << c.getArgIdx();
297 else if (c.getLoopDepth() >= 1)
298 NameOS <<
"_d" << c.getLoopDepth();
300 NameOS <<
"_s" << c.getOpPos();
303 return NameOS.str().str();
306void OpenMPDialect::initialize() {
309#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
312#define GET_ATTRDEF_LIST
313#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
316#define GET_TYPEDEF_LIST
317#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
320 declarePromisedInterface<ConvertToLLVMPatternInterface, OpenMPDialect>();
322 MemRefType::attachInterface<MemRefPointerLikeModel>(*
getContext());
323 LLVM::LLVMPointerType::attachInterface<LLVMPointerPointerLikeModel>(
328 mlir::ModuleOp::attachInterface<mlir::omp::OffloadModuleDefaultModel>(
334 mlir::LLVM::GlobalOp::attachInterface<
337 mlir::LLVM::LLVMFuncOp::attachInterface<
340 mlir::func::FuncOp::attachInterface<
349 if (!isa<DeclareTargetInterface>(op))
350 return op->
emitError() <<
"omp.declare_target can only be applied to "
351 "DeclareTargetInterface ops";
353 auto declareTargetAttr = dyn_cast<DeclareTargetAttr>(attr);
354 if (!declareTargetAttr)
356 <<
"omp.declare_target must be an #omp.declaretarget attribute";
358 if (isa<mlir::FunctionOpInterface>(op)) {
359 if (declareTargetAttr.getAutomap())
361 <<
"omp.declare_target 'automap' is not valid on functions";
364 if (declareTargetAttr.getCaptureClause() ==
365 mlir::omp::DeclareTargetCaptureClause::link)
367 <<
"omp.declare_target 'link' is not valid on functions";
370 if (declareTargetAttr.getImplicit())
372 <<
"omp.declare_target 'implicit' is only valid on functions";
378OpenMPDialect::verifyOperationAttribute(
Operation *op,
380 if (attribute.
getName() ==
"omp.declare_target")
408 allocatorVars.push_back(operand);
409 allocatorTypes.push_back(type);
415 allocateVars.push_back(operand);
416 allocateTypes.push_back(type);
427 for (
unsigned i = 0; i < allocateVars.size(); ++i) {
428 std::string separator = i == allocateVars.size() - 1 ?
"" :
", ";
429 p << allocatorVars[i] <<
" : " << allocatorTypes[i] <<
" -> ";
430 p << allocateVars[i] <<
" : " << allocateTypes[i] << separator;
438template <
typename ClauseAttr>
440 using ClauseT =
decltype(std::declval<ClauseAttr>().getValue());
445 if (std::optional<ClauseT> enumValue = symbolizeEnum<ClauseT>(enumStr)) {
446 attr = ClauseAttr::get(parser.
getContext(), *enumValue);
449 return parser.
emitError(loc,
"invalid clause value: '") << enumStr <<
"'";
452template <
typename ClauseAttr>
454 p << stringifyEnum(attr.getValue());
480 std::optional<omp::LinearModifier> linearModifier;
482 linearModifier = omp::LinearModifier::val;
484 linearModifier = omp::LinearModifier::ref;
486 linearModifier = omp::LinearModifier::uval;
489 bool hasLinearModifierParens = linearModifier.has_value();
490 if (hasLinearModifierParens && parser.
parseLParen())
498 if (hasLinearModifierParens && parser.
parseRParen())
501 linearVars.push_back(var);
502 linearTypes.push_back(type);
503 linearStepVars.push_back(stepVar);
504 linearStepTypes.push_back(stepType);
505 if (linearModifier) {
507 omp::LinearModifierAttr::get(parser.
getContext(), *linearModifier));
509 modifiers.push_back(UnitAttr::get(parser.
getContext()));
515 linearModifiers = ArrayAttr::get(parser.
getContext(), modifiers);
525 size_t linearVarsSize = linearVars.size();
526 for (
unsigned i = 0; i < linearVarsSize; ++i) {
530 Attribute modAttr = linearModifiers ? linearModifiers[i] :
nullptr;
531 auto mod = modAttr ? dyn_cast<omp::LinearModifierAttr>(modAttr) :
nullptr;
533 p << omp::stringifyLinearModifier(mod.getValue()) <<
"(";
535 p << linearVars[i] <<
" : " << linearTypes[i];
536 p <<
" = " << linearStepVars[i] <<
" : " << stepVarTypes[i];
552 if (!linearModifiers)
554 if (linearModifiers->size() != linearVars.size())
556 <<
"expected as many linear modifiers as linear variables";
557 if (!isDeclareSimd) {
558 for (
Attribute attr : *linearModifiers) {
561 auto modAttr = dyn_cast<omp::LinearModifierAttr>(attr);
564 omp::LinearModifier mod = modAttr.getValue();
565 if (mod == omp::LinearModifier::ref || mod == omp::LinearModifier::uval)
567 <<
"linear modifier '" << omp::stringifyLinearModifier(mod)
568 <<
"' may only be specified on a declare simd directive";
583 for (
const auto &it : nontemporalVars)
584 if (!nontemporalItems.insert(it).second)
585 return op->
emitOpError() <<
"nontemporal variable used more than once";
594 std::optional<ArrayAttr> alignments,
597 if (!alignedVars.empty()) {
598 if (!alignments || alignments->size() != alignedVars.size())
600 <<
"expected as many alignment values as aligned variables";
603 return op->
emitOpError() <<
"unexpected alignment values attribute";
609 for (
auto it : alignedVars)
610 if (!alignedItems.insert(it).second)
611 return op->
emitOpError() <<
"aligned variable used more than once";
617 for (
unsigned i = 0; i < (*alignments).size(); ++i) {
618 if (
auto intAttr = llvm::dyn_cast<IntegerAttr>((*alignments)[i])) {
619 if (intAttr.getValue().sle(0))
620 return op->
emitOpError() <<
"alignment should be greater than 0";
622 return op->
emitOpError() <<
"expected integer alignment";
639 if (parser.parseOperand(alignedVars.emplace_back()) ||
640 parser.parseColonType(alignedTypes.emplace_back()) ||
641 parser.parseArrow() ||
642 parser.parseAttribute(alignmentVec.emplace_back())) {
649 alignmentsAttr = ArrayAttr::get(parser.getContext(), alignments);
656 std::optional<ArrayAttr> alignments) {
657 for (
unsigned i = 0; i < alignedVars.size(); ++i) {
660 p << alignedVars[i] <<
" : " << alignedVars[i].
getType();
661 p <<
" -> " << (*alignments)[i];
669 ArrayAttr privateSyms =
nullptr,
bool requirePrivateIndices =
false) {
670 if (allocateVars.size() != allocatorVars.size())
672 "expected equal sizes for allocate and allocator variables");
674 if (allocateVars.empty()) {
675 if (allocateAlignments)
677 "unexpected allocate alignments without allocate variables");
678 if (allocatePrivateIndices)
680 "unexpected allocate private indices without allocate variables");
684 if (allocateAlignments) {
686 if (alignments.size() != allocateVars.size())
688 "expected as many allocate alignments as allocate variables");
689 for (
int64_t alignment : alignments) {
691 return op->
emitError(
"expected non-negative allocate alignments");
692 if (alignment != 0 && (alignment & (alignment - 1)) != 0)
694 "expected positive allocate alignments to be powers of two");
698 if (!allocatePrivateIndices) {
699 if (requirePrivateIndices)
701 "expected an allocate private index for each allocate variable");
706 if (
indices.size() != allocateVars.size())
708 "expected as many allocate private indices as allocate variables");
711 for (
auto [allocateVar, privateIndex] :
712 llvm::zip_equal(allocateVars,
indices)) {
713 if (privateIndex < 0 ||
714 static_cast<uint64_t
>(privateIndex) >= privateVars.size())
715 return op->
emitError(
"allocate private index is out of range");
716 if (!usedPrivateSlots.insert(privateIndex).second)
718 "allocate private index refers to a private variable more than once");
720 Value privateVar = privateVars[privateIndex];
721 if (allocateVar.getType() != privateVar.
getType())
723 <<
"type mismatch between allocate variable and private variable "
726 if (allocateVar != privateVar)
728 <<
"allocate variable does not match private variable at index "
732 static_cast<uint64_t
>(privateIndex) >= privateSyms.size())
734 "allocate private index does not have a privatizer symbol");
736 auto privateSym = dyn_cast<SymbolRefAttr>(privateSyms[privateIndex]);
739 "allocate private index does not reference a privatizer symbol");
740 PrivateClauseOp privatizer =
743 return op->
emitError() <<
"failed to lookup privatizer op with symbol: '"
744 << privateSym <<
"'";
745 if (privatizer.getDataSharingType() != DataSharingClauseType::Private &&
746 privatizer.getDataSharingType() != DataSharingClauseType::FirstPrivate)
748 "allocate private index must refer to private or firstprivate "
762 if (modifiers.size() > 2)
764 for (
const auto &mod : modifiers) {
767 auto symbol = symbolizeScheduleModifier(mod);
770 <<
" unknown modifier type: " << mod;
775 if (modifiers.size() == 1) {
776 if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd) {
777 modifiers.push_back(modifiers[0]);
778 modifiers[0] = stringifyScheduleModifier(ScheduleModifier::none);
780 }
else if (modifiers.size() == 2) {
783 if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd ||
784 symbolizeScheduleModifier(modifiers[1]) != ScheduleModifier::simd)
786 <<
" incorrect modifier order";
802 ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd,
803 std::optional<OpAsmParser::UnresolvedOperand> &chunkSize,
808 std::optional<mlir::omp::ClauseScheduleKind> schedule =
809 symbolizeClauseScheduleKind(keyword);
813 scheduleAttr = ClauseScheduleKindAttr::get(parser.
getContext(), *schedule);
815 case ClauseScheduleKind::Static:
816 case ClauseScheduleKind::Dynamic:
817 case ClauseScheduleKind::Guided:
823 chunkSize = std::nullopt;
826 case ClauseScheduleKind::Auto:
827 case ClauseScheduleKind::Runtime:
828 case ClauseScheduleKind::Distribute:
829 chunkSize = std::nullopt;
838 modifiers.push_back(mod);
844 if (!modifiers.empty()) {
846 if (std::optional<ScheduleModifier> mod =
847 symbolizeScheduleModifier(modifiers[0])) {
848 scheduleMod = ScheduleModifierAttr::get(parser.
getContext(), *mod);
850 return parser.
emitError(loc,
"invalid schedule modifier");
853 if (modifiers.size() > 1) {
854 assert(symbolizeScheduleModifier(modifiers[1]) == ScheduleModifier::simd);
864 ClauseScheduleKindAttr scheduleKind,
865 ScheduleModifierAttr scheduleMod,
866 UnitAttr scheduleSimd,
Value scheduleChunk,
867 Type scheduleChunkType) {
868 p << stringifyClauseScheduleKind(scheduleKind.getValue());
870 p <<
" = " << scheduleChunk <<
" : " << scheduleChunk.
getType();
872 p <<
", " << stringifyScheduleModifier(scheduleMod.getValue());
884 ClauseOrderKindAttr &order,
885 OrderModifierAttr &orderMod) {
890 if (std::optional<OrderModifier> enumValue =
891 symbolizeOrderModifier(enumStr)) {
892 orderMod = OrderModifierAttr::get(parser.
getContext(), *enumValue);
899 if (std::optional<ClauseOrderKind> enumValue =
900 symbolizeClauseOrderKind(enumStr)) {
901 order = ClauseOrderKindAttr::get(parser.
getContext(), *enumValue);
904 return parser.
emitError(loc,
"invalid clause value: '") << enumStr <<
"'";
908 ClauseOrderKindAttr order,
909 OrderModifierAttr orderMod) {
911 p << stringifyOrderModifier(orderMod.getValue()) <<
":";
913 p << stringifyClauseOrderKind(order.getValue());
916template <
typename ClauseTypeAttr,
typename ClauseType>
919 std::optional<OpAsmParser::UnresolvedOperand> &operand,
921 std::optional<ClauseType> (*symbolizeClause)(StringRef),
922 StringRef clauseName) {
925 if (std::optional<ClauseType> enumValue = symbolizeClause(enumStr)) {
926 prescriptiveness = ClauseTypeAttr::get(parser.
getContext(), *enumValue);
931 <<
"invalid " << clauseName <<
" modifier : '" << enumStr <<
"'";
941 <<
"expected " << clauseName <<
" operand";
944 if (operand.has_value()) {
952template <
typename ClauseTypeAttr,
typename ClauseType>
955 ClauseTypeAttr prescriptiveness,
Value operand,
957 StringRef (*stringifyClauseType)(ClauseType)) {
959 if (prescriptiveness)
960 p << stringifyClauseType(prescriptiveness.getValue()) <<
", ";
963 p << operand <<
": " << operandType;
973 std::optional<OpAsmParser::UnresolvedOperand> &grainsize,
974 Type &grainsizeType) {
976 parser, grainsizeMod, grainsize, grainsizeType,
977 &symbolizeClauseGrainsizeType,
"grainsize");
981 ClauseGrainsizeTypeAttr grainsizeMod,
984 p, op, grainsizeMod, grainsize, grainsizeType,
985 &stringifyClauseGrainsizeType);
995 std::optional<OpAsmParser::UnresolvedOperand> &numTasks,
996 Type &numTasksType) {
998 parser, numTasksMod, numTasks, numTasksType, &symbolizeClauseNumTasksType,
1003 ClauseNumTasksTypeAttr numTasksMod,
1006 p, op, numTasksMod, numTasks, numTasksType, &stringifyClauseNumTasksType);
1022 return mlir::failure();
1023 inTypeAttr = TypeAttr::get(inType);
1052 if (!typeparams.empty()) {
1053 p <<
'(' << typeparams <<
" : " << typeparamsTypes <<
')';
1055 for (
auto sh :
shape) {
1067 FallbackModifierAttr fallback,
1068 Value dynGroupprivateSize) {
1069 if (!dynGroupprivateSize && (accessGroup || fallback))
1070 return op->
emitOpError(
"dyn_groupprivate modifiers require a size operand");
1076 OpAsmParser &parser, AccessGroupModifierAttr &accessGroupAttr,
1077 FallbackModifierAttr &fallbackAttr,
1078 std::optional<OpAsmParser::UnresolvedOperand> &dynGroupprivateSize,
1081 bool parsedAccessGroup =
false;
1082 bool parsedFallback =
false;
1083 bool parsedSize =
false;
1088 if (parsedAccessGroup)
1090 "duplicate access group modifier");
1091 accessGroupAttr = AccessGroupModifierAttr::get(
1092 parser.
getContext(), AccessGroupModifier::cgroup);
1093 parsedAccessGroup =
true;
1100 "duplicate fallback modifier");
1103 "expected '(' after 'fallback'");
1104 llvm::StringRef fbKind;
1108 "expected fallback modifier (abort/null/default_mem)");
1109 std::optional<FallbackModifier> fbEnum;
1110 if (fbKind ==
"abort")
1111 fbEnum = FallbackModifier::abort;
1112 else if (fbKind ==
"null")
1113 fbEnum = FallbackModifier::null;
1114 else if (fbKind ==
"default_mem")
1115 fbEnum = FallbackModifier::default_mem;
1118 "invalid fallback modifier '" + fbKind +
"'");
1119 fallbackAttr = FallbackModifierAttr::get(parser.
getContext(), *fbEnum);
1122 "expected ')' after fallback modifier");
1123 parsedFallback =
true;
1131 "duplicate size operand");
1132 dynGroupprivateSize = operand;
1136 "expected ':' and type after size operand");
1140 "expected dyn_groupprivate_size operand");
1145 AccessGroupModifierAttr modifierFirst,
1146 FallbackModifierAttr modifierSecond,
1147 Value dynGroupprivateSize,
1150 bool needsComma =
false;
1152 if (modifierFirst) {
1153 printer << modifierFirst.getValue();
1157 if (modifierSecond) {
1160 printer <<
"fallback(";
1161 printer << modifierSecond.getValue();
1166 if (dynGroupprivateSize) {
1169 printer << dynGroupprivateSize <<
" : " << sizeType;
1189 isByRefVec.push_back(parser.parseOptionalKeyword(
"byref").succeeded());
1190 if (parser.parseAttribute(symbolVec.emplace_back()) ||
1191 parser.parseOperand(inReductionVars.emplace_back()))
1201 [&]() { return parser.parseType(inReductionTypes.emplace_back()); }))
1204 if (inReductionVars.size() != inReductionTypes.size())
1209 inReductionSyms = ArrayAttr::get(parser.
getContext(), symbolAttrs);
1226 syms = ArrayAttr::get(ctx, values);
1235 llvm::interleaveComma(
1236 llvm::zip_equal(inReductionVars, syms.getValue(), byref.
asArrayRef()), p,
1238 auto [var, sym, isByRef] = t;
1246 llvm::interleaveComma(inReductionTypes, p);
1254struct MapParseArgs {
1255 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1256 SmallVectorImpl<Type> &types;
1257 MapParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1258 SmallVectorImpl<Type> &types)
1259 : vars(vars), types(types) {}
1261struct PrivateParseArgs {
1262 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1263 llvm::SmallVectorImpl<Type> &types;
1265 UnitAttr &needsBarrier;
1267 PrivateParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1268 SmallVectorImpl<Type> &types,
ArrayAttr &syms,
1269 UnitAttr &needsBarrier,
1271 : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
1272 mapIndices(mapIndices) {}
1275struct LinearParseArgs {
1276 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1277 SmallVectorImpl<Type> &types;
1278 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars;
1279 SmallVectorImpl<Type> &stepTypes;
1283 LinearParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1284 SmallVectorImpl<Type> &types,
1285 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars,
1286 SmallVectorImpl<Type> &stepTypes,
ArrayAttr &varTypes,
1288 : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
1289 varTypes(varTypes), modifiers(modifiers), syms(syms) {}
1292struct ReductionParseArgs {
1293 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1294 SmallVectorImpl<Type> &types;
1298 ReductionParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1301 : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
1304struct AllRegionParseArgs {
1305 std::optional<MapParseArgs> hasDeviceAddrArgs;
1306 std::optional<MapParseArgs> hostEvalArgs;
1307 std::optional<ReductionParseArgs> inReductionArgs;
1308 std::optional<LinearParseArgs> linearArgs;
1309 std::optional<MapParseArgs> mapArgs;
1310 std::optional<PrivateParseArgs> privateArgs;
1311 std::optional<ReductionParseArgs> reductionArgs;
1312 std::optional<ReductionParseArgs> taskReductionArgs;
1313 std::optional<MapParseArgs> useDeviceAddrArgs;
1314 std::optional<MapParseArgs> useDevicePtrArgs;
1319 return "private_barrier";
1329 UnitAttr *needsBarrier =
nullptr) {
1333 unsigned regionArgOffset = regionPrivateArgs.size();
1341 ReductionModifier mod = ReductionModifier::defaultmod;
1342 if (succeeded(parser.parseOptionalKeyword(
"task")))
1343 mod = ReductionModifier::task;
1344 else if (succeeded(parser.parseOptionalKeyword(
"inscan")))
1345 mod = ReductionModifier::inscan;
1346 else if (succeeded(parser.parseOptionalKeyword(
"defaultmod")))
1347 mod = ReductionModifier::defaultmod;
1348 modifiers.push_back(
1349 ReductionModifierAttr::get(parser.getContext(), mod));
1352 isByRefVec.push_back(
1370 mapIndicesVec.push_back(-1);
1379 return parser.parseType(types.emplace_back());
1383 if (llvm::any_of(modifiers, [](
Attribute attr) {
1384 return cast<ReductionModifierAttr>(attr).getValue() !=
1385 ReductionModifier::defaultmod;
1387 *modifier = ArrayAttr::get(parser.
getContext(), modifiers);
1389 if (operands.size() != types.size())
1398 *needsBarrier = mlir::UnitAttr::get(parser.
getContext());
1401 auto *argsBegin = regionPrivateArgs.begin();
1403 argsBegin + regionArgOffset + types.size());
1404 for (
auto [prv, type] : llvm::zip_equal(argsSubrange, types)) {
1410 *symbols = ArrayAttr::get(parser.
getContext(), symbolAttrs);
1413 if (!mapIndicesVec.empty())
1426 StringRef keyword, std::optional<MapParseArgs> mapArgs) {
1441 StringRef keyword, std::optional<PrivateParseArgs> privateArgs) {
1447 parser, privateArgs->vars, privateArgs->types, entryBlockArgs,
1448 &privateArgs->syms, privateArgs->mapIndices,
nullptr,
1449 nullptr, &privateArgs->needsBarrier)))
1458 StringRef keyword, std::optional<ReductionParseArgs> reductionArgs) {
1463 parser, reductionArgs->vars, reductionArgs->types, entryBlockArgs,
1464 &reductionArgs->syms,
nullptr, &reductionArgs->byref,
1465 reductionArgs->modifier)))
1476 StringRef keyword, std::optional<LinearParseArgs> linearArgs) {
1484 std::optional<LinearModifier> modifier;
1485 if (succeeded(parser.parseOptionalKeyword(
"val")))
1486 modifier = LinearModifier::val;
1487 else if (succeeded(parser.parseOptionalKeyword(
"ref")))
1488 modifier = LinearModifier::ref;
1489 else if (succeeded(parser.parseOptionalKeyword(
"uval")))
1490 modifier = LinearModifier::uval;
1491 if (modifier && parser.parseLParen())
1495 OpAsmParser::Argument &arg = entryBlockArgs.emplace_back();
1496 if (parser.parseAttribute(sym) ||
1497 parser.parseOperand(linearArgs->vars.emplace_back()) ||
1498 parser.parseArrow() || parser.parseArgument(arg) ||
1499 parser.parseColonType(linearArgs->types.emplace_back()) ||
1500 parser.parseEqual() ||
1501 parser.parseOperand(linearArgs->stepVars.emplace_back()) ||
1502 parser.parseColonType(linearArgs->stepTypes.emplace_back()))
1504 arg.type = linearArgs->types.back();
1506 if (modifier && parser.parseRParen())
1509 syms.push_back(sym);
1510 modifiers.push_back(
1511 modifier ? Attribute(LinearModifierAttr::get(ctx, *modifier))
1512 : Attribute(UnitAttr::get(ctx)));
1518 linearArgs->syms = ArrayAttr::get(ctx, syms);
1519 linearArgs->modifiers = ArrayAttr::get(ctx, modifiers);
1529 AllRegionParseArgs args) {
1533 args.hasDeviceAddrArgs)))
1535 <<
"invalid `has_device_addr` format";
1538 args.hostEvalArgs)))
1540 <<
"invalid `host_eval` format";
1543 args.inReductionArgs)))
1545 <<
"invalid `in_reduction` format";
1550 <<
"invalid `linear` format";
1555 <<
"invalid `map_entries` format";
1560 <<
"invalid `private` format";
1563 args.reductionArgs)))
1565 <<
"invalid `reduction` format";
1568 args.taskReductionArgs)))
1570 <<
"invalid `task_reduction` format";
1573 args.useDeviceAddrArgs)))
1575 <<
"invalid `use_device_addr` format";
1578 args.useDevicePtrArgs)))
1580 <<
"invalid `use_device_addr` format";
1582 return parser.
parseRegion(region, entryBlockArgs);
1598 AllRegionParseArgs args;
1599 args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
1600 args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
1601 args.mapArgs.emplace(mapVars, mapTypes);
1602 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1603 privateNeedsBarrier, &privateMaps);
1614 UnitAttr &privateNeedsBarrier) {
1615 AllRegionParseArgs args;
1616 args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
1617 inReductionByref, inReductionSyms);
1618 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1619 privateNeedsBarrier);
1630 UnitAttr &privateNeedsBarrier,
ArrayAttr &reductionMods,
1634 AllRegionParseArgs args;
1635 args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
1636 inReductionByref, inReductionSyms);
1637 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1638 privateNeedsBarrier);
1639 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1640 reductionSyms, &reductionMods);
1648 UnitAttr &privateNeedsBarrier) {
1649 AllRegionParseArgs args;
1650 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1651 privateNeedsBarrier);
1659 UnitAttr &privateNeedsBarrier,
ArrayAttr &reductionMods,
1663 AllRegionParseArgs args;
1664 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1665 privateNeedsBarrier);
1666 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1667 reductionSyms, &reductionMods);
1680 UnitAttr &privateNeedsBarrier,
ArrayAttr &reductionMods,
1684 AllRegionParseArgs args;
1685 args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
1686 linearStepTypes, linearVarTypes, linearModifiers,
1688 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1689 privateNeedsBarrier);
1690 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1691 reductionSyms, &reductionMods);
1700 AllRegionParseArgs args;
1701 args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,
1702 taskReductionByref, taskReductionSyms);
1712 AllRegionParseArgs args;
1713 args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);
1714 args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);
1723struct MapPrintArgs {
1728struct PrivatePrintArgs {
1732 UnitAttr needsBarrier;
1736 : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
1737 mapIndices(mapIndices) {}
1739struct LinearPrintArgs {
1750 : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
1751 varTypes(varTypes), modifiers(modifiers), syms(syms) {}
1753struct ReductionPrintArgs {
1761 : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
1763struct AllRegionPrintArgs {
1764 std::optional<MapPrintArgs> hasDeviceAddrArgs;
1765 std::optional<MapPrintArgs> hostEvalArgs;
1766 std::optional<ReductionPrintArgs> inReductionArgs;
1767 std::optional<LinearPrintArgs> linearArgs;
1768 std::optional<MapPrintArgs> mapArgs;
1769 std::optional<PrivatePrintArgs> privateArgs;
1770 std::optional<ReductionPrintArgs> reductionArgs;
1771 std::optional<ReductionPrintArgs> taskReductionArgs;
1772 std::optional<MapPrintArgs> useDeviceAddrArgs;
1773 std::optional<MapPrintArgs> useDevicePtrArgs;
1782 UnitAttr needsBarrier =
nullptr) {
1783 if (argsSubrange.empty())
1786 p << clauseName <<
"(";
1790 symbols = ArrayAttr::get(ctx, values);
1804 llvm::interleaveComma(
1805 llvm::zip_equal(operands, argsSubrange, symbols, mapIndices.asArrayRef(),
1806 byref.asArrayRef()),
1808 auto [op, arg, sym, map, isByRef] = t;
1810 ReductionModifier mod =
1811 cast<ReductionModifierAttr>(modifier[index]).getValue();
1812 if (mod != ReductionModifier::defaultmod)
1813 p << stringifyReductionModifier(mod) <<
" ";
1820 p << op <<
" -> " << arg;
1822 p <<
" [map_idx=" << map <<
"]";
1825 llvm::interleaveComma(types, p);
1833 StringRef clauseName,
ValueRange argsSubrange,
1834 std::optional<MapPrintArgs> mapArgs) {
1841 StringRef clauseName,
ValueRange argsSubrange,
1842 std::optional<PrivatePrintArgs> privateArgs) {
1845 p, ctx, clauseName, argsSubrange, privateArgs->vars, privateArgs->types,
1846 privateArgs->syms, privateArgs->mapIndices,
nullptr,
1847 nullptr, privateArgs->needsBarrier);
1853 std::optional<ReductionPrintArgs> reductionArgs) {
1856 reductionArgs->vars, reductionArgs->types,
1857 reductionArgs->syms,
nullptr,
1858 reductionArgs->byref, reductionArgs->modifier);
1862 StringRef clauseName,
ValueRange argsSubrange,
1863 std::optional<LinearPrintArgs> linearArgs) {
1864 if (!linearArgs || argsSubrange.empty())
1867 p << clauseName <<
"(";
1868 for (
auto [i, var] : llvm::enumerate(linearArgs->vars)) {
1872 linearArgs->modifiers
1873 ? dyn_cast_if_present<LinearModifierAttr>(linearArgs->modifiers[i])
1876 p << stringifyLinearModifier(modifier.getValue()) <<
"(";
1877 if (linearArgs->syms)
1878 p << linearArgs->syms[i] <<
" ";
1879 p << var <<
" -> " << argsSubrange[i] <<
" : " << linearArgs->types[i]
1880 <<
" = " << linearArgs->stepVars[i] <<
" : " << linearArgs->stepTypes[i];
1886 if (linearArgs->varTypes)
1887 p <<
"linear_var_types(" << linearArgs->varTypes <<
") ";
1891 const AllRegionPrintArgs &args) {
1892 auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op);
1896 iface.getHasDeviceAddrBlockArgs(),
1897 args.hasDeviceAddrArgs);
1901 args.inReductionArgs);
1909 args.reductionArgs);
1911 iface.getTaskReductionBlockArgs(),
1912 args.taskReductionArgs);
1914 iface.getUseDeviceAddrBlockArgs(),
1915 args.useDeviceAddrArgs);
1917 iface.getUseDevicePtrBlockArgs(), args.useDevicePtrArgs);
1931 UnitAttr privateNeedsBarrier,
1933 AllRegionPrintArgs args;
1934 args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
1935 args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
1936 args.mapArgs.emplace(mapVars, mapTypes);
1937 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1938 privateNeedsBarrier, privateMaps);
1946 ArrayAttr privateSyms, UnitAttr privateNeedsBarrier) {
1947 AllRegionPrintArgs args;
1948 args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
1949 inReductionByref, inReductionSyms);
1950 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1951 privateNeedsBarrier,
1960 ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
1963 AllRegionPrintArgs args;
1964 args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
1965 inReductionByref, inReductionSyms);
1966 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1967 privateNeedsBarrier,
1969 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1970 reductionSyms, reductionMods);
1977 UnitAttr privateNeedsBarrier) {
1978 AllRegionPrintArgs args;
1979 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1980 privateNeedsBarrier,
1990 AllRegionPrintArgs args;
1991 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1992 privateNeedsBarrier,
1994 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1995 reductionSyms, reductionMods);
2004 UnitAttr privateNeedsBarrier,
ArrayAttr reductionMods,
2007 AllRegionPrintArgs args;
2008 args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
2009 linearStepTypes, linearVarTypes, linearModifiers,
2011 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
2012 privateNeedsBarrier,
2014 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
2015 reductionSyms, reductionMods);
2025 AllRegionPrintArgs args;
2026 args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,
2027 taskReductionByref, taskReductionSyms);
2037 AllRegionPrintArgs args;
2038 args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);
2039 args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);
2043template <
typename ParsePrefixFn>
2052 if (failed(parsePrefix()))
2060 if (llvm::isa<mlir::omp::IteratedType>(ty)) {
2061 iteratedVars.push_back(v);
2062 iteratedTypes.push_back(ty);
2064 plainVars.push_back(v);
2065 plainTypes.push_back(ty);
2071template <
typename Pr
intPrefixFn>
2075 PrintPrefixFn &&printPrefixForPlain,
2076 PrintPrefixFn &&printPrefixForIterated) {
2083 p << v <<
" : " << t;
2087 for (
unsigned i = 0; i < iteratedVars.size(); ++i)
2088 emit(iteratedVars[i], iteratedTypes[i], printPrefixForIterated);
2089 for (
unsigned i = 0; i < plainVars.size(); ++i)
2090 emit(plainVars[i], plainTypes[i], printPrefixForPlain);
2097 if (reductionMods && reductionMods.size() != reductionVars.size())
2099 "expected as many reduction modifiers as reduction variables");
2111 if (!reductionVars.empty()) {
2112 if (!reductionSyms || reductionSyms->size() != reductionVars.size())
2114 <<
"expected as many reduction symbol references "
2115 "as reduction variables";
2116 if (reductionByref && reductionByref->size() != reductionVars.size())
2117 return op->
emitError() <<
"expected as many reduction variable by "
2118 "reference attributes as reduction variables";
2121 return op->
emitOpError() <<
"unexpected reduction symbol references";
2128 for (
auto args : llvm::zip(reductionVars, *reductionSyms)) {
2129 Value accum = std::get<0>(args);
2131 if (!accumulators.insert(accum).second)
2132 return op->
emitOpError() <<
"accumulator variable used more than once";
2135 auto symbolRef = llvm::cast<SymbolRefAttr>(std::get<1>(args));
2139 return op->
emitOpError() <<
"expected symbol reference " << symbolRef
2140 <<
" to point to a reduction declaration";
2142 if (decl.getAccumulatorType() && decl.getAccumulatorType() != varType)
2144 <<
"expected accumulator (" << varType
2145 <<
") to be the same type as reduction declaration ("
2146 << decl.getAccumulatorType() <<
")";
2165 if (parser.parseOperand(copyprivateVars.emplace_back()) ||
2166 parser.parseArrow() ||
2167 parser.parseAttribute(symsVec.emplace_back()) ||
2168 parser.parseColonType(copyprivateTypes.emplace_back()))
2174 copyprivateSyms = ArrayAttr::get(parser.
getContext(), syms);
2182 std::optional<ArrayAttr> copyprivateSyms) {
2183 if (!copyprivateSyms.has_value())
2185 llvm::interleaveComma(
2186 llvm::zip(copyprivateVars, *copyprivateSyms, copyprivateTypes), p,
2187 [&](
const auto &args) {
2188 p << std::get<0>(args) <<
" -> " << std::get<1>(args) <<
" : "
2189 << std::get<2>(args);
2196 std::optional<ArrayAttr> copyprivateSyms) {
2197 size_t copyprivateSymsSize =
2198 copyprivateSyms.has_value() ? copyprivateSyms->size() : 0;
2199 if (copyprivateSymsSize != copyprivateVars.size())
2200 return op->
emitOpError() <<
"inconsistent number of copyprivate vars (= "
2201 << copyprivateVars.size()
2202 <<
") and functions (= " << copyprivateSymsSize
2203 <<
"), both must be equal";
2204 if (!copyprivateSyms.has_value())
2207 for (
auto copyprivateVarAndSym :
2208 llvm::zip(copyprivateVars, *copyprivateSyms)) {
2210 llvm::cast<SymbolRefAttr>(std::get<1>(copyprivateVarAndSym));
2211 std::optional<std::variant<mlir::func::FuncOp, mlir::LLVM::LLVMFuncOp>>
2213 if (mlir::func::FuncOp mlirFuncOp =
2216 funcOp = mlirFuncOp;
2217 else if (mlir::LLVM::LLVMFuncOp llvmFuncOp =
2220 funcOp = llvmFuncOp;
2222 auto getNumArguments = [&] {
2223 return std::visit([](
auto &f) {
return f.getNumArguments(); }, *funcOp);
2226 auto getArgumentType = [&](
unsigned i) {
2227 return std::visit([i](
auto &f) {
return f.getArgumentTypes()[i]; },
2232 return op->
emitOpError() <<
"expected symbol reference " << symbolRef
2233 <<
" to point to a copy function";
2235 if (getNumArguments() != 2)
2237 <<
"expected copy function " << symbolRef <<
" to have 2 operands";
2239 Type argTy = getArgumentType(0);
2240 if (argTy != getArgumentType(1))
2241 return op->
emitOpError() <<
"expected copy function " << symbolRef
2242 <<
" arguments to have the same type";
2244 Type varType = std::get<0>(copyprivateVarAndSym).getType();
2245 if (argTy != varType)
2247 <<
"expected copy function arguments' type (" << argTy
2248 <<
") to be the same as copyprivate variable's type (" << varType
2273 OpAsmParser::UnresolvedOperand operand;
2275 if (parser.parseKeyword(&keyword) || parser.parseArrow() ||
2276 parser.parseOperand(operand) || parser.parseColonType(ty))
2278 std::optional<ClauseTaskDepend> keywordDepend =
2279 symbolizeClauseTaskDepend(keyword);
2283 ClauseTaskDependAttr::get(parser.getContext(), *keywordDepend);
2284 if (llvm::isa<mlir::omp::IteratedType>(ty)) {
2285 iteratedVars.push_back(operand);
2286 iteratedTypes.push_back(ty);
2287 iterKindsVec.push_back(kindAttr);
2289 dependVars.push_back(operand);
2290 dependTypes.push_back(ty);
2291 kindsVec.push_back(kindAttr);
2297 dependKinds = ArrayAttr::get(parser.
getContext(), kinds);
2299 iteratedKinds = ArrayAttr::get(parser.
getContext(), iterKinds);
2306 std::optional<ArrayAttr> dependKinds,
2309 std::optional<ArrayAttr> iteratedKinds) {
2312 std::optional<ArrayAttr> kinds) {
2313 for (
unsigned i = 0, e = vars.size(); i < e; ++i) {
2316 p << stringifyClauseTaskDepend(
2317 llvm::cast<mlir::omp::ClauseTaskDependAttr>((*kinds)[i])
2319 <<
" -> " << vars[i] <<
" : " << types[i];
2323 printEntries(dependVars, dependTypes, dependKinds);
2324 printEntries(iteratedVars, iteratedTypes, iteratedKinds);
2329 std::optional<ArrayAttr> dependKinds,
2331 std::optional<ArrayAttr> iteratedKinds,
2333 if (!dependVars.empty()) {
2334 if (!dependKinds || dependKinds->size() != dependVars.size())
2335 return op->
emitOpError() <<
"expected as many depend values"
2336 " as depend variables";
2338 if (dependKinds && !dependKinds->empty())
2339 return op->
emitOpError() <<
"unexpected depend values";
2342 if (!iteratedVars.empty()) {
2343 if (!iteratedKinds || iteratedKinds->size() != iteratedVars.size())
2344 return op->
emitOpError() <<
"expected as many depend iterated values"
2345 " as depend iterated variables";
2347 if (iteratedKinds && !iteratedKinds->empty())
2348 return op->
emitOpError() <<
"unexpected depend iterated values";
2363 IntegerAttr &hintAttr) {
2364 StringRef hintKeyword;
2370 auto parseKeyword = [&]() -> ParseResult {
2373 if (hintKeyword ==
"uncontended")
2375 else if (hintKeyword ==
"contended")
2377 else if (hintKeyword ==
"nonspeculative")
2379 else if (hintKeyword ==
"speculative")
2383 << hintKeyword <<
" is not a valid hint";
2394 IntegerAttr hintAttr) {
2395 int64_t hint = hintAttr.getInt();
2403 auto bitn = [](
int value,
int n) ->
bool {
return value & (1 << n); };
2405 bool uncontended = bitn(hint, 0);
2406 bool contended = bitn(hint, 1);
2407 bool nonspeculative = bitn(hint, 2);
2408 bool speculative = bitn(hint, 3);
2412 hints.push_back(
"uncontended");
2414 hints.push_back(
"contended");
2416 hints.push_back(
"nonspeculative");
2418 hints.push_back(
"speculative");
2420 llvm::interleaveComma(hints, p);
2427 auto bitn = [](
int value,
int n) ->
bool {
return value & (1 << n); };
2429 bool uncontended = bitn(hint, 0);
2430 bool contended = bitn(hint, 1);
2431 bool nonspeculative = bitn(hint, 2);
2432 bool speculative = bitn(hint, 3);
2434 if (uncontended && contended)
2435 return op->
emitOpError() <<
"the hints omp_sync_hint_uncontended and "
2436 "omp_sync_hint_contended cannot be combined";
2437 if (nonspeculative && speculative)
2438 return op->
emitOpError() <<
"the hints omp_sync_hint_nonspeculative and "
2439 "omp_sync_hint_speculative cannot be combined.";
2450 return (value & flag) == flag;
2458static ParseResult parseMapClause(
OpAsmParser &parser,
2459 ClauseMapFlagsAttr &mapType) {
2460 ClauseMapFlags mapTypeBits = ClauseMapFlags::none;
2463 auto parseTypeAndMod = [&]() -> ParseResult {
2464 StringRef mapTypeMod;
2468 if (mapTypeMod ==
"always")
2469 mapTypeBits |= ClauseMapFlags::always;
2471 if (mapTypeMod ==
"implicit")
2472 mapTypeBits |= ClauseMapFlags::implicit;
2474 if (mapTypeMod ==
"ompx_hold")
2475 mapTypeBits |= ClauseMapFlags::ompx_hold;
2477 if (mapTypeMod ==
"close")
2478 mapTypeBits |= ClauseMapFlags::close;
2480 if (mapTypeMod ==
"present")
2481 mapTypeBits |= ClauseMapFlags::present;
2483 if (mapTypeMod ==
"to")
2484 mapTypeBits |= ClauseMapFlags::to;
2486 if (mapTypeMod ==
"from")
2487 mapTypeBits |= ClauseMapFlags::from;
2489 if (mapTypeMod ==
"tofrom")
2490 mapTypeBits |= ClauseMapFlags::to | ClauseMapFlags::from;
2492 if (mapTypeMod ==
"delete")
2493 mapTypeBits |= ClauseMapFlags::del;
2495 if (mapTypeMod ==
"storage")
2496 mapTypeBits |= ClauseMapFlags::storage;
2498 if (mapTypeMod ==
"return_param")
2499 mapTypeBits |= ClauseMapFlags::return_param;
2501 if (mapTypeMod ==
"private")
2502 mapTypeBits |= ClauseMapFlags::priv;
2504 if (mapTypeMod ==
"literal")
2505 mapTypeBits |= ClauseMapFlags::literal;
2507 if (mapTypeMod ==
"attach")
2508 mapTypeBits |= ClauseMapFlags::attach;
2510 if (mapTypeMod ==
"attach_always")
2511 mapTypeBits |= ClauseMapFlags::attach_always;
2513 if (mapTypeMod ==
"attach_never")
2514 mapTypeBits |= ClauseMapFlags::attach_never;
2516 if (mapTypeMod ==
"attach_auto")
2517 mapTypeBits |= ClauseMapFlags::attach_auto;
2519 if (mapTypeMod ==
"ref_ptr")
2520 mapTypeBits |= ClauseMapFlags::ref_ptr;
2522 if (mapTypeMod ==
"ref_ptee")
2523 mapTypeBits |= ClauseMapFlags::ref_ptee;
2525 if (mapTypeMod ==
"is_device_ptr")
2526 mapTypeBits |= ClauseMapFlags::is_device_ptr;
2528 if (mapTypeMod ==
"target_param")
2529 mapTypeBits |= ClauseMapFlags::target_param;
2546 ClauseMapFlagsAttr mapType) {
2548 ClauseMapFlags mapFlags = mapType.getValue();
2553 mapTypeStrs.push_back(
"always");
2555 mapTypeStrs.push_back(
"implicit");
2557 mapTypeStrs.push_back(
"ompx_hold");
2559 mapTypeStrs.push_back(
"close");
2561 mapTypeStrs.push_back(
"present");
2563 mapTypeStrs.push_back(
"target_param");
2572 mapTypeStrs.push_back(
"tofrom");
2574 mapTypeStrs.push_back(
"from");
2576 mapTypeStrs.push_back(
"to");
2579 mapTypeStrs.push_back(
"delete");
2581 mapTypeStrs.push_back(
"return_param");
2583 mapTypeStrs.push_back(
"storage");
2585 mapTypeStrs.push_back(
"private");
2587 mapTypeStrs.push_back(
"literal");
2589 mapTypeStrs.push_back(
"attach");
2591 mapTypeStrs.push_back(
"attach_always");
2593 mapTypeStrs.push_back(
"attach_never");
2595 mapTypeStrs.push_back(
"attach_auto");
2597 mapTypeStrs.push_back(
"ref_ptr");
2599 mapTypeStrs.push_back(
"ref_ptee");
2601 mapTypeStrs.push_back(
"is_device_ptr");
2602 if (mapFlags == ClauseMapFlags::none)
2603 mapTypeStrs.push_back(
"none");
2605 for (
unsigned int i = 0; i < mapTypeStrs.size(); ++i) {
2606 p << mapTypeStrs[i];
2607 if (i + 1 < mapTypeStrs.size()) {
2613static ParseResult parseMembersIndex(
OpAsmParser &parser,
2617 auto parseIndices = [&]() -> ParseResult {
2622 APInt(64, value,
false)));
2636 memberIdxs.push_back(ArrayAttr::get(parser.
getContext(), values));
2640 if (!memberIdxs.empty())
2641 membersIdx = ArrayAttr::get(parser.
getContext(), memberIdxs);
2651 llvm::interleaveComma(membersIdx, p, [&p](
Attribute v) {
2653 auto memberIdx = cast<ArrayAttr>(v);
2654 llvm::interleaveComma(memberIdx.getValue(), p, [&p](
Attribute v2) {
2655 p << cast<IntegerAttr>(v2).getInt();
2662 VariableCaptureKindAttr mapCaptureType) {
2663 std::string typeCapStr;
2664 llvm::raw_string_ostream typeCap(typeCapStr);
2665 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByRef)
2667 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByCopy)
2668 typeCap <<
"ByCopy";
2669 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::VLAType)
2670 typeCap <<
"VLAType";
2671 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::This)
2677 VariableCaptureKindAttr &mapCaptureType) {
2678 StringRef mapCaptureKey;
2682 if (mapCaptureKey ==
"This")
2683 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2684 parser.
getContext(), mlir::omp::VariableCaptureKind::This);
2685 if (mapCaptureKey ==
"ByRef")
2686 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2687 parser.
getContext(), mlir::omp::VariableCaptureKind::ByRef);
2688 if (mapCaptureKey ==
"ByCopy")
2689 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2690 parser.
getContext(), mlir::omp::VariableCaptureKind::ByCopy);
2691 if (mapCaptureKey ==
"VLAType")
2692 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2693 parser.
getContext(), mlir::omp::VariableCaptureKind::VLAType);
2699 Operation *op, mlir::omp::MapInfoOp mapInfoOp,
2703 mlir::omp::ClauseMapFlags mapTypeBits = mapInfoOp.getMapType();
2706 bool from =
mapTypeToBool(mapTypeBits, ClauseMapFlags::from);
2709 bool always =
mapTypeToBool(mapTypeBits, ClauseMapFlags::always);
2710 bool close =
mapTypeToBool(mapTypeBits, ClauseMapFlags::close);
2711 bool implicit =
mapTypeToBool(mapTypeBits, ClauseMapFlags::implicit);
2712 bool attach =
mapTypeToBool(mapTypeBits, ClauseMapFlags::attach);
2714 if ((isa<TargetDataOp>(op) || isa<TargetOp>(op)) && del)
2716 "to, from, tofrom and alloc map types are permitted");
2718 if (isa<TargetEnterDataOp>(op) && (from || del))
2719 return emitError(op->
getLoc(),
"to and alloc map types are permitted");
2721 if (isa<TargetExitDataOp>(op) && to)
2723 "from, release and delete map types are permitted");
2725 if (isa<TargetUpdateOp>(op)) {
2728 "at least one of to or from map types must be "
2729 "specified, other map types are not permitted");
2732 if (!to && !from && !attach) {
2734 "at least one of to or from or attach map types must be "
2735 "specified, other map types are not permitted");
2738 auto updateVar = mapInfoOp.getVarPtr();
2740 if ((to && from) || (to && updateFromVars.contains(updateVar)) ||
2741 (from && updateToVars.contains(updateVar))) {
2744 "either to or from map types can be specified, not both");
2747 if (always || close || implicit) {
2750 "present, mapper and iterator map type modifiers are permitted");
2756 to ? updateToVars.insert(updateVar) : updateFromVars.insert(updateVar);
2760 if ((mapInfoOp.getVarPtrPtr() && !mapInfoOp.getVarPtrPtrType()) ||
2761 (!mapInfoOp.getVarPtrPtr() && mapInfoOp.getVarPtrPtrType())) {
2763 "if varPtrPtr or varPtrPtrType is specified, then both "
2775 for (
auto mapOp : mapVars) {
2776 if (!mapOp.getDefiningOp())
2779 if (
auto mapInfoOp = mapOp.getDefiningOp<mlir::omp::MapInfoOp>()) {
2783 }
else if (!isa<DeclareMapperInfoOp>(op)) {
2785 "map argument is not a map entry operation");
2790 for (
auto iterVal : mapIterated) {
2791 auto iterOp = iterVal.getDefiningOp<mlir::omp::IteratorOp>();
2793 return op->
emitOpError() <<
"'map_iterated' arguments must be defined by "
2794 "'omp.iterator' ops";
2797 mlir::omp::YieldOp yieldOp = iterOp.getYieldOp();
2798 auto yieldedMapInfo =
2799 yieldOp && yieldOp.getNumOperands() == 1
2800 ? yieldOp.getResults()[0].getDefiningOp<mlir::omp::MapInfoOp>()
2802 if (!yieldedMapInfo)
2803 return op->
emitOpError() <<
"'map_iterated' iterator body must yield "
2804 "a value defined by 'omp.map.info'";
2814template <
typename OpType>
2818 std::optional<DenseI64ArrayAttr> privateMapIndices =
2819 targetOp.getPrivateMapsAttr();
2822 if (!privateMapIndices.has_value() || !privateMapIndices.value())
2827 if (privateMapIndices.value().size() !=
2828 static_cast<int64_t>(privateVars.size()))
2829 return emitError(targetOp.getLoc(),
"sizes of `private` operand range and "
2830 "`private_maps` attribute mismatch");
2840 StringRef clauseName,
2842 for (
Value var : vars)
2843 if (!llvm::isa_and_present<MapInfoOp>(var.getDefiningOp()))
2845 <<
"'" << clauseName
2846 <<
"' arguments must be defined by 'omp.map.info' ops";
2850LogicalResult MapInfoOp::verify() {
2851 if (getMapperId() &&
2853 *
this, getMapperIdAttr())) {
2868 const TargetDataOperands &clauses) {
2869 TargetDataOp::build(builder, state, clauses.device, clauses.ifExpr,
2870 clauses.mapVars, clauses.mapIterated,
2871 clauses.useDeviceAddrVars, clauses.useDevicePtrVars);
2874LogicalResult TargetDataOp::verify() {
2875 if (getMapVars().empty() && getMapIterated().empty() &&
2876 getUseDevicePtrVars().empty() && getUseDeviceAddrVars().empty()) {
2877 return ::emitError(this->getLoc(),
2878 "At least one of map, use_device_ptr_vars, or "
2879 "use_device_addr_vars operand must be present");
2883 getUseDevicePtrVars())))
2887 getUseDeviceAddrVars())))
2897void TargetEnterDataOp::build(
2901 TargetEnterDataOp::build(
2903 clauses.dependVars,
makeArrayAttr(ctx, clauses.dependIteratedKinds),
2904 clauses.dependIterated, clauses.device, clauses.ifExpr, clauses.mapVars,
2905 clauses.mapIterated, clauses.nowait);
2908LogicalResult TargetEnterDataOp::verify() {
2909 LogicalResult verifyDependVars =
2911 getDependIteratedKinds(), getDependIterated());
2912 return failed(verifyDependVars)
2924 TargetExitDataOp::build(
2926 clauses.dependVars,
makeArrayAttr(ctx, clauses.dependIteratedKinds),
2927 clauses.dependIterated, clauses.device, clauses.ifExpr, clauses.mapVars,
2928 clauses.mapIterated, clauses.nowait);
2931LogicalResult TargetExitDataOp::verify() {
2932 LogicalResult verifyDependVars =
2934 getDependIteratedKinds(), getDependIterated());
2935 return failed(verifyDependVars)
2947 TargetUpdateOp::build(builder, state,
makeArrayAttr(ctx, clauses.dependKinds),
2950 clauses.dependIterated, clauses.device, clauses.ifExpr,
2951 clauses.mapVars, clauses.mapIterated, clauses.nowait);
2954LogicalResult TargetUpdateOp::verify() {
2955 LogicalResult verifyDependVars =
2957 getDependIteratedKinds(), getDependIterated());
2958 return failed(verifyDependVars)
2971 builder, state, clauses.allocateVars, clauses.allocatorVars,
2974 makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,
2975 makeArrayAttr(ctx, clauses.dependIteratedKinds), clauses.dependIterated,
2976 clauses.device, clauses.dynGroupprivateAccessGroup,
2977 clauses.dynGroupprivateFallback, clauses.dynGroupprivateSize,
2978 clauses.hasDeviceAddrVars, clauses.hostEvalVars, clauses.ifExpr,
2979 clauses.inReductionVars,
2981 makeArrayAttr(ctx, clauses.inReductionSyms), clauses.isDevicePtrVars,
2982 clauses.mapVars, clauses.mapIterated, clauses.nowait, clauses.privateVars,
2983 makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
2984 clauses.threadLimitVars,
nullptr, clauses.
kernelType);
2987bool TargetOp::hasHostEvalTripCount() {
2988 TargetExecMode mode = getKernelType();
2989 if (mode == TargetExecMode::spmd || mode == TargetExecMode::spmd_no_loop)
2992 if (mode == TargetExecMode::bare)
2998 cast<ComposableOpInterface>(getOperation()).findCapturedOp();
2999 if (
auto loopNestOp = dyn_cast_if_present<LoopNestOp>(capturedOp)) {
3001 loopNestOp.gatherWrappers(loopWrappers);
3003 LoopWrapperInterface *innermostWrapper = loopWrappers.begin();
3004 if (isa<SimdOp>(innermostWrapper))
3005 innermostWrapper = std::next(innermostWrapper);
3007 auto numWrappers = std::distance(innermostWrapper, loopWrappers.end());
3008 if (numWrappers != 1)
3011 if (!isa<DistributeOp>(innermostWrapper))
3015 if (isa_and_present<TeamsOp>(parentOp) &&
3031 if (mapVarPtr == inReductionVar)
3037LogicalResult TargetOp::verify() {
3039 getOperation(), getAllocateVars(), getAllocatorVars(),
3040 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3041 getPrivateVars(), getPrivateSymsAttr())))
3044 if (getKernelType() == TargetExecMode::bare && !isCombined())
3045 return emitOpError() <<
"bare kernel requires 'omp.combined'";
3048 getDependIteratedKinds(),
3049 getDependIterated())))
3053 getHasDeviceAddrVars())))
3060 *
this, getDynGroupprivateAccessGroupAttr(),
3061 getDynGroupprivateFallbackAttr(), getDynGroupprivateSize())))
3068 getInReductionVars(),
3069 getInReductionByref())))
3077 for (
Value inReductionVar : getInReductionVars()) {
3078 bool captured =
false;
3079 for (
Value mapVar : getMapVars()) {
3080 auto mapInfo = mapVar.getDefiningOp<MapInfoOp>();
3087 return emitOpError() <<
"in_reduction variable must be captured by a "
3088 "matching map_entries entry";
3094LogicalResult TargetOp::verifyRegions() {
3095 auto teamsOps = getOps<TeamsOp>();
3096 auto numNestedTeams = std::distance(teamsOps.begin(), teamsOps.end());
3097 if (numNestedTeams > 1)
3098 return emitError(
"target containing multiple 'omp.teams' nested ops");
3100 if (numNestedTeams == 0) {
3101 switch (getKernelType()) {
3102 case TargetExecMode::bare:
3103 return emitOpError()
3104 <<
"bare kernel must contain a nested 'omp.teams' operation";
3105 case TargetExecMode::spmd_no_loop:
3106 return emitOpError() <<
"spmd_no_loop kernel must contain a nested "
3107 "'omp.teams' operation";
3114 cast<ComposableOpInterface>(getOperation()).findCapturedOp();
3115 if ((getKernelType() == TargetExecMode::spmd ||
3116 getKernelType() == TargetExecMode::spmd_no_loop) &&
3117 !isa_and_present<LoopNestOp>(capturedOp))
3118 return emitOpError()
3119 <<
"SPMD kernel must capture an 'omp.loop_nest' operation";
3121 bool isTargetDevice =
false;
3122 if (
auto offloadMod = (*this)->getParentOfType<OffloadModuleInterface>())
3123 if (offloadMod.getIsTargetDevice())
3124 isTargetDevice =
true;
3128 cast<BlockArgOpenMPOpInterface>(getOperation()).getHostEvalBlockArgs();
3130 bool hostEvalTripCount = hasHostEvalTripCount();
3131 for (
Value hostEvalArg : hostEvalBlockArgs) {
3133 if (
auto teamsOp = dyn_cast<TeamsOp>(user)) {
3135 if (hostEvalArg == teamsOp.getNumTeamsLower() ||
3136 llvm::is_contained(teamsOp.getNumTeamsUpperVars(), hostEvalArg) ||
3137 llvm::is_contained(teamsOp.getThreadLimitVars(), hostEvalArg))
3140 return emitOpError() <<
"host_eval argument only legal as 'num_teams' "
3141 "and 'thread_limit' in 'omp.teams'";
3143 if (
auto parallelOp = dyn_cast<ParallelOp>(user)) {
3144 if (llvm::is_contained(parallelOp.getNumThreadsVars(), hostEvalArg))
3147 return emitOpError()
3148 <<
"host_eval argument only legal as 'num_threads' in "
3151 if (
auto loopNestOp = dyn_cast<LoopNestOp>(user)) {
3152 if (hostEvalTripCount &&
3153 (llvm::is_contained(loopNestOp.getLoopLowerBounds(), hostEvalArg) ||
3154 llvm::is_contained(loopNestOp.getLoopUpperBounds(), hostEvalArg) ||
3155 llvm::is_contained(loopNestOp.getLoopSteps(), hostEvalArg)))
3158 return emitOpError() <<
"host_eval argument only legal as loop bounds "
3159 "and steps in 'omp.loop_nest' when trip count "
3160 "must be evaluated in the host";
3163 return emitOpError() <<
"host_eval argument illegal use in '"
3164 << user->getName() <<
"' operation";
3168 if (hostEvalTripCount && !isTargetDevice) {
3169 auto loopOp = cast<LoopNestOp>(capturedOp);
3170 for (
auto arg : llvm::concat<Value>(loopOp.getLoopLowerBounds(),
3171 loopOp.getLoopUpperBounds(),
3172 loopOp.getLoopSteps())) {
3173 if (!llvm::is_contained(hostEvalBlockArgs, arg))
3174 return emitOpError() <<
"nested 'omp.loop_nest' bounds expected to "
3175 "be host-evaluated";
3188 ParallelOp::build(builder, state,
ValueRange(),
3203 const ParallelOperands &clauses) {
3205 ParallelOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
3208 clauses.ifExpr, clauses.numThreadsVars, clauses.privateVars,
3210 clauses.privateNeedsBarrier, clauses.procBindKind,
3212 clauses.reductionVars,
3217template <
typename OpType>
3219 auto privateVars = op.getPrivateVars();
3220 auto privateSyms = op.getPrivateSymsAttr();
3222 if (privateVars.empty() && (privateSyms ==
nullptr || privateSyms.empty()))
3225 auto numPrivateVars = privateVars.size();
3226 auto numPrivateSyms = (privateSyms ==
nullptr) ? 0 : privateSyms.size();
3228 if (numPrivateVars != numPrivateSyms)
3229 return op.emitError() <<
"inconsistent number of private variables and "
3230 "privatizer op symbols, private vars: "
3232 <<
" vs. privatizer op symbols: " << numPrivateSyms;
3234 for (
auto privateVarInfo : llvm::zip_equal(privateVars, privateSyms)) {
3235 Type varType = std::get<0>(privateVarInfo).getType();
3236 SymbolRefAttr privateSym = cast<SymbolRefAttr>(std::get<1>(privateVarInfo));
3237 PrivateClauseOp privatizerOp =
3240 if (privatizerOp ==
nullptr)
3241 return op.emitError() <<
"failed to lookup privatizer op with symbol: '"
3242 << privateSym <<
"'";
3244 Type privatizerType = privatizerOp.getArgType();
3246 if (privatizerType && (varType != privatizerType))
3247 return op.emitError()
3248 <<
"type mismatch between a "
3249 << stringifyDataSharingClauseType(
3250 privatizerOp.getDataSharingType())
3251 <<
" variable and its privatizer op, var type: " << varType
3252 <<
" vs. privatizer op type: " << privatizerType;
3254 if (privatizerOp.getDataSharingType() == DataSharingClauseType::Linear)
3255 return op.emitError() <<
"`linear` privatizer '" << privateSym
3256 <<
"' can only be used in a `linear` clause";
3263template <
typename OpType>
3266 std::optional<ArrayAttr> linearSyms = op.getLinearSyms();
3267 if (linearVars.empty() && (!linearSyms || linearSyms->empty()))
3270 if (!linearSyms || linearSyms->size() != linearVars.size())
3271 return op.emitOpError()
3272 <<
"expected as many linear privatizers as linear variables";
3274 for (
auto [var, sym] : llvm::zip_equal(linearVars, *linearSyms)) {
3275 auto symRef = cast<SymbolRefAttr>(sym);
3279 return op.emitOpError() <<
"failed to lookup privatizer op with symbol: '"
3281 if (privatizer.getDataSharingType() != DataSharingClauseType::Linear)
3282 return op.emitOpError() <<
"privatizer '" << symRef
3283 <<
"' of a linear variable must be `linear`";
3284 Type argType = privatizer.getArgType();
3285 if (argType && argType != var.getType())
3286 return op.emitOpError()
3287 <<
"type mismatch between a linear variable and its privatizer "
3289 << var.getType() <<
" vs. privatizer op type: " << argType;
3294LogicalResult ParallelOp::verify() {
3298 getOperation(), getAllocateVars(), getAllocatorVars(),
3299 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3300 getPrivateVars(), getPrivateSymsAttr(),
3305 getReductionByref(), getReductionModsAttr());
3308LogicalResult ParallelOp::verifyRegions() {
3309 auto distChildOps = getOps<DistributeOp>();
3310 int numDistChildOps = std::distance(distChildOps.begin(), distChildOps.end());
3311 if (numDistChildOps > 1)
3313 <<
"multiple 'omp.distribute' nested inside of 'omp.parallel'";
3315 if (numDistChildOps == 1) {
3318 <<
"'omp.composite' attribute missing from composite operation";
3320 auto *ompDialect =
getContext()->getLoadedDialect<OpenMPDialect>();
3321 Operation &distributeOp = **distChildOps.begin();
3323 if (&childOp == &distributeOp || ompDialect != childOp.getDialect())
3327 return emitError() <<
"unexpected OpenMP operation inside of composite "
3329 << childOp.getName();
3331 }
else if (isComposite()) {
3333 <<
"'omp.composite' attribute present in non-composite operation";
3350 const TeamsOperands &clauses) {
3354 builder, state, clauses.allocateVars, clauses.allocatorVars,
3357 clauses.dynGroupprivateAccessGroup, clauses.dynGroupprivateFallback,
3358 clauses.dynGroupprivateSize, clauses.ifExpr, clauses.numTeamsLower,
3359 clauses.numTeamsUpperVars, {},
nullptr,
3361 makeArrayAttr(ctx, clauses.reductionMods), clauses.reductionVars,
3363 makeArrayAttr(ctx, clauses.reductionSyms), clauses.threadLimitVars);
3370 if (numTeamsLower) {
3371 if (numTeamsUpperVars.size() != 1)
3373 "expected exactly one num_teams upper bound when lower bound is "
3377 "expected num_teams upper bound and lower bound to be "
3384LogicalResult TeamsOp::verify() {
3391 auto parentTarget = llvm::dyn_cast_if_present<TargetOp>(op->
getParentOp());
3393 return emitError(
"expected to be nested inside of omp.target or not nested "
3394 "in any OpenMP dialect operations");
3398 this->getNumTeamsUpperVars())))
3402 parentTarget.getKernelType() == TargetExecMode::spmd_no_loop &&
3403 (getNumTeamsLower() || !getNumTeamsUpperVars().empty()))
3404 return emitOpError() <<
"'num_teams' not allowed in SPMD-no-loop kernels";
3407 getOperation(), getAllocateVars(), getAllocatorVars(),
3408 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3409 getPrivateVars(), getPrivateSymsAttr())))
3413 op, getDynGroupprivateAccessGroupAttr(),
3414 getDynGroupprivateFallbackAttr(), getDynGroupprivateSize())))
3421 getReductionByref(), getReductionModsAttr());
3429 return getParentOp().getPrivateVars();
3433 return getParentOp().getReductionVars();
3441 const SectionsOperands &clauses) {
3444 SectionsOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
3451 clauses.reductionVars,
3457LogicalResult SectionsOp::verify() {
3459 return emitOpError() <<
"cannot be a non-innermost combined construct leaf";
3462 getOperation(), getAllocateVars(), getAllocatorVars(),
3463 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3464 getPrivateVars(), getPrivateSymsAttr())))
3468 getReductionByref(), getReductionModsAttr());
3471LogicalResult SectionsOp::verifyRegions() {
3472 for (
auto &inst : *getRegion().begin()) {
3473 if (!(isa<SectionOp>(inst) || isa<TerminatorOp>(inst))) {
3474 return emitOpError()
3475 <<
"expected omp.section op or terminator op inside region";
3495 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(op);
3513 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(op);
3531 addEffects(iface.getReductionVars(), iface.getReductionBlockArgs());
3532 addEffects(iface.getInReductionVars(), iface.getInReductionBlockArgs());
3542void SectionsOp::getEffects(
3551 bool hasUnknownRecipeEffects =
3552 (!getPrivateVars().empty() && !getPrivateAssumeBoundedRecipeEffects()) ||
3553 (!getReductionVars().empty() &&
3554 !getReductionAssumeBoundedRecipeEffects());
3556 if (hasUnknownRecipeEffects) {
3569 const ScopeOperands &clauses) {
3572 builder, state, clauses.allocateVars, clauses.allocatorVars,
3575 clauses.nowait, clauses.privateVars,
3576 makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
3577 makeArrayAttr(ctx, clauses.reductionMods), clauses.reductionVars,
3582LogicalResult ScopeOp::verify() {
3584 getOperation(), getAllocateVars(), getAllocatorVars(),
3585 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3586 getPrivateVars(), getPrivateSymsAttr(),
3594 getReductionByref(), getReductionModsAttr());
3602 const SingleOperands &clauses) {
3606 builder, state, clauses.allocateVars, clauses.allocatorVars,
3609 clauses.copyprivateVars,
makeArrayAttr(ctx, clauses.copyprivateSyms),
3610 nullptr, clauses.nowait,
3616LogicalResult SingleOp::verify() {
3618 getOperation(), getAllocateVars(), getAllocatorVars(),
3619 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3620 getPrivateVars(), getPrivateSymsAttr())))
3624 getCopyprivateSyms());
3627void SingleOp::getEffects(
3636 bool hasUnknownRecipeEffects =
3637 (!getPrivateVars().empty() && !getPrivateAssumeBoundedRecipeEffects()) ||
3638 (!getCopyprivateVars().empty() &&
3639 !getCopyprivateAssumeBoundedRecipeEffects());
3640 if (hasUnknownRecipeEffects) {
3660 const WorkshareOperands &clauses) {
3661 WorkshareOp::build(builder, state, clauses.nowait);
3664LogicalResult WorkshareOp::verify() {
3666 return emitOpError() <<
"cannot be a non-innermost combined construct leaf";
3675LogicalResult WorkshareLoopWrapperOp::verifyRegions() {
3676 if (isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||
3678 return emitOpError() <<
"expected to be a standalone loop wrapper";
3687LogicalResult LoopWrapperInterface::verifyImpl() {
3691 return emitOpError() <<
"loop wrapper must also have the `NoTerminator` "
3692 "and `SingleBlock` traits";
3695 return emitOpError() <<
"loop wrapper does not contain exactly one region";
3698 if (range_size(region.
getOps()) != 1)
3699 return emitOpError()
3700 <<
"loop wrapper does not contain exactly one nested op";
3703 if (!isa<LoopNestOp, LoopWrapperInterface>(firstOp))
3704 return emitOpError() <<
"nested in loop wrapper is not another loop "
3705 "wrapper or `omp.loop_nest`";
3714Operation *ComposableOpInterface::findCapturedOp() {
3718 if (
auto wrapperOp = dyn_cast<LoopWrapperInterface>(op))
3719 return wrapperOp.getWrappedLoop();
3724 if (!isCombined() && !isComposite())
3729 if (
auto wrapperOp = dyn_cast<LoopWrapperInterface>(&nestedOp))
3730 return wrapperOp.getWrappedLoop();
3732 if (
auto composableOp = dyn_cast<ComposableOpInterface>(&nestedOp))
3733 return composableOp.findCapturedOp();
3742LogicalResult ComposableOpInterface::verifyImpl() {
3746 return emitOpError() <<
"composable ops must have a single region";
3748 if (isComposite() && !isa<LoopWrapperInterface, ParallelOp>(op))
3749 return emitOpError() <<
"non-loop wrapper cannot be composite";
3755 auto count = llvm::count_if(
3757 if (isa<ComposableOpInterface, LoopWrapperInterface>(op)) {
3778 return emitOpError()
3779 <<
"multiple eligible child ops found in combined op";
3790 if (successor->isReachable(parentBlock))
3791 return emitOpError() <<
"nested combined child op is part of a loop";
3795 !domInfo.
dominates(parentBlock, &block))
3796 return emitOpError()
3797 <<
"nested combined child op doesn't unconditionally execute";
3807 const LoopOperands &clauses) {
3810 LoopOp::build(builder, state, clauses.bindKind, clauses.privateVars,
3812 clauses.privateNeedsBarrier, clauses.order, clauses.orderMod,
3814 clauses.reductionVars,
3819LogicalResult LoopOp::verify() {
3824 getReductionByref(), getReductionModsAttr());
3827LogicalResult LoopOp::verifyRegions() {
3828 if (llvm::isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||
3830 return emitOpError() <<
"expected to be a standalone loop wrapper";
3841 build(builder, state, {}, {},
3846 nullptr,
false,
nullptr,
3848 nullptr, {},
nullptr,
3859 const WsloopOperands &clauses) {
3862 builder, state, clauses.allocateVars, clauses.allocatorVars,
3865 clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
3866 clauses.linearModifiers,
makeArrayAttr(ctx, clauses.linearSyms),
3867 clauses.nowait, clauses.order, clauses.orderMod, clauses.ordered,
3868 clauses.privateVars,
makeArrayAttr(ctx, clauses.privateSyms),
3869 clauses.privateNeedsBarrier,
makeArrayAttr(ctx, clauses.reductionMods),
3870 clauses.reductionVars,
3872 makeArrayAttr(ctx, clauses.reductionSyms), clauses.scheduleKind,
3873 clauses.scheduleChunk, clauses.scheduleMod, clauses.scheduleSimd);
3876LogicalResult WsloopOp::verify() {
3878 getOperation(), getAllocateVars(), getAllocatorVars(),
3879 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3880 getPrivateVars(), getPrivateSymsAttr())))
3886 if (getLinearVars().size() &&
3887 getLinearVarTypes().value().size() != getLinearVars().size())
3888 return emitError() <<
"Ill-formed type attributes for linear variables";
3894 getReductionByref(), getReductionModsAttr());
3897LogicalResult WsloopOp::verifyRegions() {
3898 bool isCompositeChildLeaf =
3899 llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
3901 if (LoopWrapperInterface nested = getNestedWrapper()) {
3904 <<
"'omp.composite' attribute missing from composite wrapper";
3908 if (!isa<SimdOp>(nested))
3909 return emitError() <<
"only supported nested wrapper is 'omp.simd'";
3911 }
else if (isComposite() && !isCompositeChildLeaf) {
3913 <<
"'omp.composite' attribute present in non-composite wrapper";
3914 }
else if (!isComposite() && isCompositeChildLeaf) {
3916 <<
"'omp.composite' attribute missing from composite wrapper";
3927 const SimdOperands &clauses) {
3930 builder, state, clauses.alignedVars,
3932 clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
3933 clauses.linearModifiers,
makeArrayAttr(ctx, clauses.linearSyms),
3934 clauses.nontemporalVars, clauses.order, clauses.orderMod,
3935 clauses.privateVars,
makeArrayAttr(ctx, clauses.privateSyms),
3936 clauses.privateNeedsBarrier,
makeArrayAttr(ctx, clauses.reductionMods),
3937 clauses.reductionVars,
3943LogicalResult SimdOp::verify() {
3945 getReductionModsAttr())))
3948 if (getSimdlen().has_value() && getSafelen().has_value() &&
3949 getSimdlen().value() > getSafelen().value())
3950 return emitOpError()
3951 <<
"simdlen clause and safelen clause are both present, but the "
3952 "simdlen value is not less than or equal to safelen value";
3964 bool isCompositeChildLeaf =
3965 llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
3967 if (!isComposite() && isCompositeChildLeaf)
3969 <<
"'omp.composite' attribute missing from composite wrapper";
3971 if (isComposite() && !isCompositeChildLeaf)
3973 <<
"'omp.composite' attribute present in non-composite wrapper";
3977 std::optional<ArrayAttr> privateSyms = getPrivateSyms();
3979 for (
const Attribute &sym : *privateSyms) {
3980 auto symRef = cast<SymbolRefAttr>(sym);
3981 omp::PrivateClauseOp privatizer =
3983 getOperation(), symRef);
3985 return emitError() <<
"Cannot find privatizer '" << symRef <<
"'";
3986 if (privatizer.getDataSharingType() ==
3987 DataSharingClauseType::FirstPrivate)
3988 return emitError() <<
"FIRSTPRIVATE cannot be used with SIMD";
3995 if (getLinearVars().size() &&
3996 getLinearVarTypes().value().size() != getLinearVars().size())
3997 return emitError() <<
"Ill-formed type attributes for linear variables";
4005 for (
Value var : getLinearVars()) {
4006 if (privateVars.contains(var) || reductionVars.contains(var))
4007 return emitOpError()
4008 <<
"linear variables cannot appear in other data-sharing clauses";
4014LogicalResult SimdOp::verifyRegions() {
4015 if (getNestedWrapper())
4016 return emitOpError() <<
"must wrap an 'omp.loop_nest' directly";
4026 const DistributeOperands &clauses) {
4027 DistributeOp::build(
4028 builder, state, clauses.allocateVars, clauses.allocatorVars,
4031 clauses.allocatePrivateIndices),
4032 clauses.distScheduleStatic, clauses.distScheduleChunkSize, clauses.order,
4033 clauses.orderMod, clauses.privateVars,
4035 clauses.privateNeedsBarrier);
4038LogicalResult DistributeOp::verify() {
4039 if (this->getDistScheduleChunkSize() && !this->getDistScheduleStatic())
4040 return emitOpError() <<
"chunk size set without "
4041 "dist_schedule_static being present";
4044 getOperation(), getAllocateVars(), getAllocatorVars(),
4045 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
4046 getPrivateVars(), getPrivateSymsAttr())))
4055LogicalResult DistributeOp::verifyRegions() {
4056 if (LoopWrapperInterface nested = getNestedWrapper()) {
4059 <<
"'omp.composite' attribute missing from composite wrapper";
4062 if (isa<WsloopOp>(nested)) {
4064 if (!llvm::dyn_cast_if_present<ParallelOp>(parentOp) ||
4065 !cast<ComposableOpInterface>(parentOp).isComposite()) {
4066 return emitError() <<
"an 'omp.wsloop' nested wrapper is only allowed "
4067 "when a composite 'omp.parallel' is the direct "
4070 }
else if (!isa<SimdOp>(nested))
4071 return emitError() <<
"only supported nested wrappers are 'omp.simd' and "
4073 }
else if (isComposite()) {
4075 <<
"'omp.composite' attribute present in non-composite wrapper";
4086 const DeclareMapperInfoOperands &clauses) {
4087 DeclareMapperInfoOp::build(builder, state, clauses.mapVars,
4088 clauses.mapIterated);
4091LogicalResult DeclareMapperInfoOp::verify() {
4095LogicalResult DeclareMapperOp::verifyRegions() {
4096 if (!llvm::isa_and_present<DeclareMapperInfoOp>(
4097 getRegion().getBlocks().front().getTerminator()))
4098 return emitOpError() <<
"expected terminator to be a DeclareMapperInfoOp";
4107LogicalResult DeclareReductionOp::verifyRegions() {
4108 if (!getAllocRegion().empty()) {
4109 for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
4110 if (yieldOp.getResults().size() != 1 ||
4111 yieldOp.getResults().getTypes()[0] !=
getType())
4112 return emitOpError() <<
"expects alloc region to yield a value "
4113 "of the reduction type";
4117 if (getInitializerRegion().empty())
4118 return emitOpError() <<
"expects non-empty initializer region";
4119 Block &initializerEntryBlock = getInitializerRegion().
front();
4122 if (!getAllocRegion().empty())
4123 return emitOpError() <<
"expects two arguments to the initializer region "
4124 "when an allocation region is used";
4126 if (getAllocRegion().empty())
4127 return emitOpError() <<
"expects one argument to the initializer region "
4128 "when no allocation region is used";
4130 return emitOpError()
4131 <<
"expects one or two arguments to the initializer region";
4135 if (arg.getType() !=
getType())
4136 return emitOpError() <<
"expects initializer region argument to match "
4137 "the reduction type";
4139 for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
4140 if (yieldOp.getResults().size() != 1 ||
4141 yieldOp.getResults().getTypes()[0] !=
getType())
4142 return emitOpError() <<
"expects initializer region to yield a value "
4143 "of the reduction type";
4146 if (getReductionRegion().empty())
4147 return emitOpError() <<
"expects non-empty reduction region";
4148 Block &reductionEntryBlock = getReductionRegion().
front();
4153 return emitOpError() <<
"expects reduction region with two arguments of "
4154 "the reduction type";
4155 for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
4156 if (yieldOp.getResults().size() != 1 ||
4157 yieldOp.getResults().getTypes()[0] !=
getType())
4158 return emitOpError() <<
"expects reduction region to yield a value "
4159 "of the reduction type";
4162 if (!getAtomicReductionRegion().empty()) {
4163 Block &atomicReductionEntryBlock = getAtomicReductionRegion().
front();
4167 return emitOpError() <<
"expects atomic reduction region with two "
4168 "arguments of the same type";
4169 auto ptrType = llvm::dyn_cast<PointerLikeType>(
4172 (ptrType.getElementType() && ptrType.getElementType() !=
getType()))
4173 return emitOpError() <<
"expects atomic reduction region arguments to "
4174 "be accumulators containing the reduction type";
4177 if (getCleanupRegion().empty())
4179 Block &cleanupEntryBlock = getCleanupRegion().
front();
4182 return emitOpError() <<
"expects cleanup region with one argument "
4183 "of the reduction type";
4193 const TaskOperands &clauses) {
4195 TaskOp::build(builder, state, clauses.iterated, clauses.affinityVars,
4196 clauses.allocateVars, clauses.allocatorVars,
4199 makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,
4201 clauses.dependIterated, clauses.final, clauses.ifExpr,
4202 clauses.inReductionVars,
4204 makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
4205 clauses.priority, clauses.privateVars,
4207 clauses.privateNeedsBarrier, clauses.threadset, clauses.untied,
4208 clauses.eventHandle);
4211LogicalResult TaskOp::verify() {
4213 getOperation(), getAllocateVars(), getAllocatorVars(),
4214 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
4215 getPrivateVars(), getPrivateSymsAttr())))
4218 LogicalResult verifyDependVars =
4220 getDependIteratedKinds(), getDependIterated());
4221 if (
failed(verifyDependVars))
4222 return verifyDependVars;
4228 getInReductionVars(), getInReductionByref());
4236 const TaskgroupOperands &clauses) {
4238 TaskgroupOp::build(builder, state, clauses.allocateVars,
4239 clauses.allocatorVars,
4242 clauses.taskReductionVars,
4248LogicalResult TaskgroupOp::verify() {
4250 getOperation(), getAllocateVars(), getAllocatorVars(),
4251 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr())))
4255 getTaskReductionVars(),
4256 getTaskReductionByref());
4261 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(op);
4262 auto taskVars = iface.getTaskReductionVars();
4263 for (
auto &source :
MutableArrayRef(taskVars.getBase(), taskVars.size())) {
4268 for (
BlockArgument arg : iface.getTaskReductionBlockArgs()) {
4276 if (!taskVars.empty()) {
4289void TaskgroupOp::getEffects(
4294 if (!getTaskReductionVars().empty() &&
4295 !getTaskReductionAssumeBoundedRecipeEffects())
4306 const TaskloopContextOperands &clauses) {
4308 TaskloopContextOp::build(
4309 builder, state, clauses.allocateVars, clauses.allocatorVars,
4312 clauses.grainsizeMod, clauses.grainsize, clauses.ifExpr,
4313 clauses.inReductionVars,
4315 makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
4316 clauses.nogroup, clauses.numTasksMod, clauses.numTasks, clauses.priority,
4317 clauses.privateVars,
4319 clauses.privateNeedsBarrier,
makeArrayAttr(ctx, clauses.reductionMods),
4320 clauses.reductionVars,
4322 makeArrayAttr(ctx, clauses.reductionSyms), clauses.threadset,
4324 state.
addAttribute(
"omp.combined", UnitAttr::get(ctx));
4327TaskloopWrapperOp TaskloopContextOp::getLoopOp() {
4328 return cast<TaskloopWrapperOp>(
4330 return isa<TaskloopWrapperOp>(op);
4334LogicalResult TaskloopContextOp::verify() {
4338 getOperation(), getAllocateVars(), getAllocatorVars(),
4339 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
4340 getPrivateVars(), getPrivateSymsAttr())))
4344 getReductionVars(), getReductionByref(),
4345 getReductionModsAttr())) ||
4347 getInReductionVars(),
4348 getInReductionByref())))
4351 if (!getReductionVars().empty() && getNogroup())
4352 return emitError(
"if a reduction clause is present on the taskloop "
4353 "directive, the nogroup clause must not be specified");
4354 for (
auto var : getReductionVars()) {
4355 if (llvm::is_contained(getInReductionVars(), var))
4356 return emitError(
"the same list item cannot appear in both a reduction "
4357 "and an in_reduction clause");
4360 if (getGrainsize() && getNumTasks()) {
4362 "the grainsize clause and num_tasks clause are mutually exclusive and "
4363 "may not appear on the same taskloop directive");
4371 return emitOpError(
"must always contain the 'omp.combined' attribute");
4376LogicalResult TaskloopContextOp::verifyRegions() {
4377 Region ®ion = getRegion();
4379 return isa<TaskloopWrapperOp>(op);
4381 if (loopWrapperIt == region.
front().
end())
4382 return emitOpError()
4383 <<
"expected a TaskloopWrapperOp directly nested in the region";
4385 auto loopWrapperOp = cast<TaskloopWrapperOp>(*loopWrapperIt);
4386 auto loopNestOp = dyn_cast<LoopNestOp>(loopWrapperOp.getWrappedLoop());
4392 std::function<
bool(
Value)> isValidBoundValue = [&](
Value value) ->
bool {
4393 Region *valueRegion = value.getParentRegion();
4399 Operation *defOp = value.getDefiningOp();
4403 return llvm::all_of(defOp->
getOperands(), isValidBoundValue);
4405 auto hasUnsupportedTaskloopLocalBound = [&](
OperandRange range) ->
bool {
4406 return llvm::any_of(range,
4407 [&](
Value value) {
return !isValidBoundValue(value); });
4410 if (hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopLowerBounds()) ||
4411 hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopUpperBounds()) ||
4412 hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopSteps())) {
4413 return emitOpError()
4414 <<
"expects loop bounds and steps to be defined outside of the "
4415 "taskloop.context region or by pure, regionless operations "
4416 "that do not depend on block arguments";
4427 const TaskloopWrapperOperands &clauses) {
4428 TaskloopWrapperOp::build(builder, state);
4431TaskloopContextOp TaskloopWrapperOp::getTaskloopContext() {
4432 return dyn_cast<TaskloopContextOp>(getOperation()->getParentOp());
4435LogicalResult TaskloopWrapperOp::verify() {
4436 TaskloopContextOp context = getTaskloopContext();
4438 return emitOpError() <<
"expected to be nested in a taskloop context op";
4442LogicalResult TaskloopWrapperOp::verifyRegions() {
4443 if (LoopWrapperInterface nested = getNestedWrapper()) {
4446 <<
"'omp.composite' attribute missing from composite wrapper";
4450 if (!isa<SimdOp>(nested))
4451 return emitError() <<
"only supported nested wrapper is 'omp.simd'";
4452 }
else if (isComposite()) {
4454 <<
"'omp.composite' attribute present in non-composite wrapper";
4478 for (
auto &iv : ivs)
4479 iv.type = loopVarType;
4484 result.addAttribute(
"loop_inclusive", UnitAttr::get(ctx));
4500 "collapse_num_loops",
4505 auto parseTiles = [&]() -> ParseResult {
4509 tiles.push_back(
tile);
4518 if (tiles.size() > 0)
4537 Region ®ion = getRegion();
4539 p <<
" (" << args <<
") : " << args[0].getType() <<
" = ("
4540 << getLoopLowerBounds() <<
") to (" << getLoopUpperBounds() <<
") ";
4541 if (getLoopInclusive())
4543 p <<
"step (" << getLoopSteps() <<
") ";
4544 if (
int64_t numCollapse = getCollapseNumLoops())
4545 if (numCollapse > 1)
4546 p <<
"collapse(" << numCollapse <<
") ";
4549 p <<
"tiles(" << tiles.value() <<
") ";
4555 const LoopNestOperands &clauses) {
4557 LoopNestOp::build(builder, state, clauses.collapseNumLoops,
4558 clauses.loopLowerBounds, clauses.loopUpperBounds,
4559 clauses.loopSteps, clauses.loopInclusive,
4563LogicalResult LoopNestOp::verify() {
4564 if (getLoopLowerBounds().empty())
4565 return emitOpError() <<
"must represent at least one loop";
4567 if (getLoopLowerBounds().size() != getIVs().size())
4568 return emitOpError() <<
"number of range arguments and IVs do not match";
4570 for (
auto [lb, iv] : llvm::zip_equal(getLoopLowerBounds(), getIVs())) {
4571 if (lb.getType() != iv.getType())
4572 return emitOpError()
4573 <<
"range argument type does not match corresponding IV type";
4576 uint64_t numIVs = getIVs().size();
4578 if (
const auto &numCollapse = getCollapseNumLoops())
4579 if (numCollapse > numIVs)
4580 return emitOpError()
4581 <<
"collapse value is larger than the number of loops";
4584 if (tiles.value().size() > numIVs)
4585 return emitOpError() <<
"too few canonical loops for tile dimensions";
4587 if (!llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp()))
4588 return emitOpError() <<
"expects parent op to be a loop wrapper";
4593void LoopNestOp::gatherWrappers(
4596 while (
auto wrapper =
4597 llvm::dyn_cast_if_present<LoopWrapperInterface>(parent)) {
4598 wrappers.push_back(wrapper);
4607std::tuple<NewCliOp, OpOperand *, OpOperand *>
4613 return {{},
nullptr,
nullptr};
4616 "Unexpected type of cli");
4622 auto op = cast<LoopTransformationInterface>(use.getOwner());
4624 unsigned opnum = use.getOperandNumber();
4625 if (op.isGeneratee(opnum)) {
4626 assert(!gen &&
"Each CLI may have at most one def");
4628 }
else if (op.isApplyee(opnum)) {
4629 assert(!cons &&
"Each CLI may have at most one consumer");
4632 llvm_unreachable(
"Unexpected operand for a CLI");
4636 return {create, gen, cons};
4642 case llvm::omp::ProcBindKind::OMP_PROC_BIND_close:
4643 return ClauseProcBindKind::Close;
4644 case llvm::omp::ProcBindKind::OMP_PROC_BIND_master:
4645 return ClauseProcBindKind::Master;
4646 case llvm::omp::ProcBindKind::OMP_PROC_BIND_primary:
4647 return ClauseProcBindKind::Primary;
4648 case llvm::omp::ProcBindKind::OMP_PROC_BIND_spread:
4649 return ClauseProcBindKind::Spread;
4650 case llvm::omp::ProcBindKind::OMP_PROC_BIND_default:
4651 case llvm::omp::ProcBindKind::OMP_PROC_BIND_unknown:
4654 llvm_unreachable(
"unexpected proc-bind kind");
4677 std::string cliName{
"cli"};
4681 .Case([&](CanonicalLoopOp op) {
4684 .Case([&](UnrollHeuristicOp op) -> std::string {
4685 llvm_unreachable(
"heuristic unrolling does not generate a loop");
4687 .Case([&](FuseOp op) -> std::string {
4688 unsigned opnum =
generator->getOperandNumber();
4691 if (op.getFirst().has_value() && opnum != op.getFirst().value())
4692 return "canonloop_fuse";
4696 .Case([&](TileOp op) -> std::string {
4697 auto [generateesFirst, generateesCount] =
4698 op.getGenerateesODSOperandIndexAndLength();
4699 unsigned firstGrid = generateesFirst;
4700 unsigned firstIntratile = generateesFirst + generateesCount / 2;
4701 unsigned end = generateesFirst + generateesCount;
4702 unsigned opnum =
generator->getOperandNumber();
4704 if (firstGrid <= opnum && opnum < firstIntratile) {
4705 unsigned gridnum = opnum - firstGrid + 1;
4706 return (
"grid" + Twine(gridnum)).str();
4708 if (firstIntratile <= opnum && opnum < end) {
4709 unsigned intratilenum = opnum - firstIntratile + 1;
4710 return (
"intratile" + Twine(intratilenum)).str();
4712 llvm_unreachable(
"Unexpected generatee argument");
4714 .DefaultUnreachable(
"TODO: Custom name for this operation");
4717 setNameFn(
result, cliName);
4720LogicalResult NewCliOp::verify() {
4721 Value cli = getResult();
4724 "Unexpected type of cli");
4730 auto op = dyn_cast<mlir::omp::LoopTransformationInterface>(use.getOwner());
4733 emitOpError(
"CLI must only be used by loop transformations");
4738 unsigned opnum = use.getOperandNumber();
4739 if (op.isGeneratee(opnum)) {
4742 emitOpError(
"CLI must have at most one generator");
4744 .
append(
"first generator here:");
4746 .
append(
"second generator here:");
4751 }
else if (op.isApplyee(opnum)) {
4754 emitOpError(
"CLI must have at most one consumer");
4756 .
append(
"first consumer here:")
4760 .
append(
"second consumer here:")
4767 llvm_unreachable(
"Unexpected operand for a CLI");
4775 .
append(
"see consumer here: ")
4798 setNameFn(&getRegion().front(),
"body_entry");
4801void CanonicalLoopOp::getAsmBlockArgumentNames(
Region ®ion,
4809 p <<
'(' << getCli() <<
')';
4810 p <<
' ' << getInductionVar() <<
" : " << getInductionVar().getType()
4811 <<
" in range(" << getTripCount() <<
") ";
4821 CanonicalLoopInfoType cliType =
4822 CanonicalLoopInfoType::get(parser.
getContext());
4847 if (parser.
parseRegion(*region, {inductionVariable}))
4852 result.operands.append(cliOperand);
4858 return mlir::success();
4861LogicalResult CanonicalLoopOp::verify() {
4864 if (!getRegion().empty()) {
4865 Region ®ion = getRegion();
4868 "Canonical loop region must have exactly one argument");
4872 "Region argument must be the same type as the trip count");
4878Value CanonicalLoopOp::getInductionVar() {
return getRegion().getArgument(0); }
4880std::pair<unsigned, unsigned>
4881CanonicalLoopOp::getApplyeesODSOperandIndexAndLength() {
4886std::pair<unsigned, unsigned>
4887CanonicalLoopOp::getGenerateesODSOperandIndexAndLength() {
4888 return getODSOperandIndexAndLength(odsIndex_cli);
4902 p <<
'(' << getApplyee() <<
')';
4909 auto cliType = CanonicalLoopInfoType::get(parser.
getContext());
4932 return mlir::success();
4935std::pair<unsigned, unsigned>
4936UnrollHeuristicOp ::getApplyeesODSOperandIndexAndLength() {
4937 return getODSOperandIndexAndLength(odsIndex_applyee);
4940std::pair<unsigned, unsigned>
4941UnrollHeuristicOp::getGenerateesODSOperandIndexAndLength() {
4955 p <<
'(' << getApplyee() <<
')';
4962 auto cliType = CanonicalLoopInfoType::get(parser.
getContext());
4985 return mlir::success();
4988std::pair<unsigned, unsigned>
4989UnrollFullOp::getApplyeesODSOperandIndexAndLength() {
4990 return getODSOperandIndexAndLength(odsIndex_applyee);
4993std::pair<unsigned, unsigned>
4994UnrollFullOp::getGenerateesODSOperandIndexAndLength() {
4998LogicalResult UnrollFullOp::verify() {
4999 auto [create, gen, cons] =
decodeCli(getApplyee());
5001 return emitOpError() <<
"applyee CLI has no generator";
5005 if (
auto loop = dyn_cast<CanonicalLoopOp>(gen->getOwner())) {
5007 return emitOpError() <<
"applyee loop must have a constant trip count";
5019 uint64_t unrollFactor) {
5026 p <<
'(' << getApplyee() <<
')';
5029 attrs.emplace_back(getUnrollFactorAttrName(), getUnrollFactorAttr());
5036 auto cliType = CanonicalLoopInfoType::get(parser.
getContext());
5053 return mlir::success();
5056std::pair<unsigned, unsigned>
5057UnrollPartialOp::getApplyeesODSOperandIndexAndLength() {
5058 return getODSOperandIndexAndLength(odsIndex_applyee);
5061std::pair<unsigned, unsigned>
5062UnrollPartialOp::getGenerateesODSOperandIndexAndLength() {
5073 if (!generatees.empty())
5074 p <<
'(' << llvm::interleaved(generatees) <<
')';
5076 if (!applyees.empty())
5077 p <<
" <- (" << llvm::interleaved(applyees) <<
')';
5119 bool isOnlyCanonLoops =
true;
5121 for (
Value applyee : op.getApplyees()) {
5122 auto [create, gen, cons] =
decodeCli(applyee);
5125 return op.emitOpError() <<
"applyee CLI has no generator";
5127 auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());
5128 canonLoops.push_back(loop);
5130 isOnlyCanonLoops =
false;
5135 if (!isOnlyCanonLoops)
5139 for (
auto i : llvm::seq<int>(1, canonLoops.size())) {
5140 auto parentLoop = canonLoops[i - 1];
5141 auto loop = canonLoops[i];
5143 if (parentLoop.getOperation() != loop.getOperation()->getParentOp())
5144 return op.emitOpError()
5145 <<
"tiled loop nest must be nested within each other";
5147 parentIVs.insert(parentLoop.getInductionVar());
5152 bool isPerfectlyNested = [&]() {
5153 auto &parentBody = parentLoop.getRegion();
5154 if (!parentBody.hasOneBlock())
5156 auto &parentBlock = parentBody.getBlocks().
front();
5158 auto nestedLoopIt = parentBlock.
begin();
5159 if (nestedLoopIt == parentBlock.
end() ||
5160 (&*nestedLoopIt != loop.getOperation()))
5163 auto termIt = std::next(nestedLoopIt);
5164 if (termIt == parentBlock.
end() || !isa<TerminatorOp>(termIt))
5167 if (std::next(termIt) != parentBlock.
end())
5172 if (!isPerfectlyNested)
5173 return op.emitOpError() <<
"tiled loop nest must be perfectly nested";
5175 if (parentIVs.contains(loop.getTripCount()))
5176 return op.emitOpError() <<
"tiled loop nest must be rectangular";
5193LogicalResult TileOp::verify() {
5194 if (getApplyees().empty())
5195 return emitOpError() <<
"must apply to at least one loop";
5197 if (getSizes().size() != getApplyees().size())
5198 return emitOpError() <<
"there must be one tile size for each applyee";
5200 if (!getGeneratees().empty() &&
5201 2 * getSizes().size() != getGeneratees().size())
5202 return emitOpError()
5203 <<
"expecting two times the number of generatees than applyees";
5208std::pair<unsigned, unsigned> TileOp ::getApplyeesODSOperandIndexAndLength() {
5209 return getODSOperandIndexAndLength(odsIndex_applyees);
5212std::pair<unsigned, unsigned> TileOp::getGenerateesODSOperandIndexAndLength() {
5213 return getODSOperandIndexAndLength(odsIndex_generatees);
5223 if (!generatees.empty())
5224 p <<
'(' << llvm::interleaved(generatees) <<
')';
5226 if (!applyees.empty())
5227 p <<
" <- (" << llvm::interleaved(applyees) <<
')';
5230LogicalResult FuseOp::verify() {
5231 if (getApplyees().size() < 2)
5232 return emitOpError() <<
"must apply to at least two loops";
5234 if (getFirst().has_value() && getCount().has_value()) {
5235 int64_t first = getFirst().value();
5236 int64_t count = getCount().value();
5237 if ((
unsigned)(first + count - 1) > getApplyees().size())
5238 return emitOpError() <<
"the numbers of applyees must be at least first "
5239 "minus one plus count attributes";
5240 if (!getGeneratees().empty() &&
5241 getGeneratees().size() != getApplyees().size() + 1 - count)
5242 return emitOpError() <<
"the number of generatees must be the number of "
5243 "aplyees plus one minus count";
5246 if (!getGeneratees().empty() && getGeneratees().size() != 1)
5247 return emitOpError()
5248 <<
"in a complete fuse the number of generatees must be exactly 1";
5250 for (
auto &&applyee : getApplyees()) {
5251 auto [create, gen, cons] =
decodeCli(applyee);
5254 return emitOpError() <<
"applyee CLI has no generator";
5255 auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());
5257 return emitOpError()
5258 <<
"currently only supports omp.canonical_loop as applyee";
5262std::pair<unsigned, unsigned> FuseOp::getApplyeesODSOperandIndexAndLength() {
5263 return getODSOperandIndexAndLength(odsIndex_applyees);
5266std::pair<unsigned, unsigned> FuseOp::getGenerateesODSOperandIndexAndLength() {
5267 return getODSOperandIndexAndLength(odsIndex_generatees);
5275 const CriticalDeclareOperands &clauses) {
5276 CriticalDeclareOp::build(builder, state, clauses.symName,
5277 clauses.symVisibility, clauses.hint);
5280LogicalResult CriticalDeclareOp::verify() {
5284LogicalResult CriticalOp::verify() {
5285 SymbolRefAttr currentName = getNameAttr();
5287 CriticalOp parentCritical = (*this)->getParentOfType<CriticalOp>();
5289 while (parentCritical) {
5290 SymbolRefAttr parentName = parentCritical.getNameAttr();
5292 if (currentName == parentName) {
5294 return emitOpError() <<
"cannot be nested inside another omp.critical "
5295 "region with the same name ("
5296 << currentName <<
")";
5298 return emitOpError() <<
"cannot be nested inside another unnamed "
5299 "omp.critical region";
5303 parentCritical = parentCritical->getParentOfType<CriticalOp>();
5310 if (getNameAttr()) {
5311 SymbolRefAttr symbolRef = getNameAttr();
5315 return emitOpError() <<
"expected symbol reference " << symbolRef
5316 <<
" to point to a critical declaration";
5327LogicalResult ErrorOp::verify() {
5328 if (getMessage() && getMessageExpr())
5329 return emitOpError() <<
"the message must be provided either as a constant "
5330 "`message` attribute or as a `message_expr` "
5331 "operand, but not both";
5348 return op.
emitOpError() <<
"must be nested inside of a loop";
5352 if (
auto wsloopOp = dyn_cast<WsloopOp>(wrapper)) {
5353 IntegerAttr orderedAttr = wsloopOp.getOrderedAttr();
5355 return op.
emitOpError() <<
"the enclosing worksharing-loop region must "
5356 "have an ordered clause";
5358 if (hasRegion && orderedAttr.getInt() != 0)
5359 return op.
emitOpError() <<
"the enclosing loop's ordered clause must not "
5360 "have a parameter present";
5362 if (!hasRegion && orderedAttr.getInt() == 0)
5363 return op.
emitOpError() <<
"the enclosing loop's ordered clause must "
5364 "have a parameter present";
5365 }
else if (!isa<SimdOp>(wrapper)) {
5366 return op.
emitOpError() <<
"must be nested inside of a worksharing, simd "
5367 "or worksharing simd loop";
5373 const OrderedOperands &clauses) {
5374 OrderedOp::build(builder, state, clauses.doacrossDependType,
5375 clauses.doacrossNumLoops, clauses.doacrossDependVars);
5378LogicalResult OrderedOp::verify() {
5382 auto wrapper = (*this)->getParentOfType<WsloopOp>();
5383 if (!wrapper || *wrapper.getOrdered() != *getDoacrossNumLoops())
5384 return emitOpError() <<
"number of variables in depend clause does not "
5385 <<
"match number of iteration variables in the "
5392 const OrderedRegionOperands &clauses) {
5393 OrderedRegionOp::build(builder, state, clauses.parLevelSimd);
5403 const TaskwaitOperands &clauses) {
5419LogicalResult AtomicReadOp::verify() {
5420 if (verifyCommon().
failed())
5421 return mlir::failure();
5424 if (
auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5425 if (
Attribute verAttr = moduleOp->getDiscardableAttr(
"omp.version"))
5426 version = llvm::cast<VersionAttr>(verAttr).getVersion();
5428 if (
auto mo = getMemoryOrder()) {
5429 if (*mo == ClauseMemoryOrderKind::Release) {
5430 return emitError(
"memory-order must not be release for atomic reads");
5432 if (*mo == ClauseMemoryOrderKind::Acq_rel) {
5435 return emitError(
"memory-order must not be acq_rel for atomic reads");
5445LogicalResult AtomicWriteOp::verify() {
5446 if (verifyCommon().
failed())
5447 return mlir::failure();
5450 if (
auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5451 if (
Attribute verAttr = moduleOp->getDiscardableAttr(
"omp.version"))
5452 version = llvm::cast<VersionAttr>(verAttr).getVersion();
5454 if (
auto mo = getMemoryOrder()) {
5455 if (*mo == ClauseMemoryOrderKind::Acquire) {
5456 return emitError(
"memory-order must not be acquire for atomic writes");
5458 if (*mo == ClauseMemoryOrderKind::Acq_rel) {
5461 return emitError(
"memory-order must not be acq_rel for atomic writes");
5471LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,
5477 if (
Value writeVal = op.getWriteOpVal()) {
5479 op, op.getX(), writeVal, op.getHintAttr(), op.getMemoryOrderAttr());
5485LogicalResult AtomicUpdateOp::verify() {
5486 if (verifyCommon().
failed())
5487 return mlir::failure();
5490 if (
auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5491 if (
Attribute verAttr = moduleOp->getDiscardableAttr(
"omp.version"))
5492 version = llvm::cast<VersionAttr>(verAttr).getVersion();
5494 if (
auto mo = getMemoryOrder()) {
5495 if (*mo == ClauseMemoryOrderKind::Acq_rel ||
5496 *mo == ClauseMemoryOrderKind::Acquire) {
5500 "memory-order must not be acq_rel or acquire for atomic updates");
5507LogicalResult AtomicUpdateOp::verifyRegions() {
return verifyRegionsCommon(); }
5513AtomicReadOp AtomicCaptureOp::getAtomicReadOp() {
5514 if (
auto op = dyn_cast<AtomicReadOp>(getFirstOp()))
5516 return dyn_cast<AtomicReadOp>(getSecondOp());
5519AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() {
5520 if (
auto op = dyn_cast<AtomicWriteOp>(getFirstOp()))
5522 return dyn_cast<AtomicWriteOp>(getSecondOp());
5525AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() {
5526 if (
auto op = dyn_cast<AtomicUpdateOp>(getFirstOp()))
5528 return dyn_cast<AtomicUpdateOp>(getSecondOp());
5531AtomicCompareOp AtomicCaptureOp::getAtomicCompareOp() {
5532 if (
auto op = dyn_cast<AtomicCompareOp>(getFirstOp()))
5534 return dyn_cast<AtomicCompareOp>(getSecondOp());
5537LogicalResult AtomicCaptureOp::verify() {
5541LogicalResult AtomicCaptureOp::verifyRegions() {
5542 if (verifyRegionsCommon().
failed())
5543 return mlir::failure();
5545 if (getFirstOp()->getInherentAttr(
"hint").value_or(
Attribute{}) ||
5546 getSecondOp()->getInherentAttr(
"hint").value_or(
Attribute{}))
5548 "operations inside capture region must not have hint clause");
5550 if (getFirstOp()->getInherentAttr(
"memory_order").value_or(
Attribute{}) ||
5551 getSecondOp()->getInherentAttr(
"memory_order").value_or(
Attribute{}))
5553 "operations inside capture region must not have memory_order clause");
5561LogicalResult AtomicCompareOp::verify() {
5562 if (verifyCommon().
failed())
5563 return mlir::failure();
5567 if (
auto failOrder = getFailMemoryOrder()) {
5568 if (*failOrder != ClauseMemoryOrderKind::Seq_cst &&
5569 *failOrder != ClauseMemoryOrderKind::Acquire &&
5570 *failOrder != ClauseMemoryOrderKind::Relaxed)
5572 "fail_memory_order must be 'seq_cst', 'acquire' or 'relaxed'");
5577LogicalResult AtomicCompareOp::verifyRegions() {
5578 if (verifyRegionsCommon().
failed())
5579 return mlir::failure();
5581 if (verifyOperator().
failed())
5582 return mlir::failure();
5587 if (!terminator || !isa<YieldOp>(terminator))
5588 return emitOpError(
"region must be terminated with omp.yield");
5598 const CancelOperands &clauses) {
5599 CancelOp::build(builder, state, clauses.cancelDirective, clauses.ifExpr);
5612LogicalResult CancelOp::verify() {
5613 ClauseCancellationConstructType cct = getCancelDirective();
5616 if (!structuralParent)
5617 return emitOpError() <<
"Orphaned cancel construct";
5619 if ((cct == ClauseCancellationConstructType::Parallel) &&
5620 !mlir::isa<ParallelOp>(structuralParent)) {
5621 return emitOpError() <<
"cancel parallel must appear "
5622 <<
"inside a parallel region";
5624 if (cct == ClauseCancellationConstructType::Loop) {
5627 auto wsloopOp = mlir::dyn_cast<WsloopOp>(structuralParent->
getParentOp());
5630 return emitOpError()
5631 <<
"cancel loop must appear inside a worksharing-loop region";
5633 if (wsloopOp.getNowaitAttr()) {
5634 return emitError() <<
"A worksharing construct that is canceled "
5635 <<
"must not have a nowait clause";
5637 if (wsloopOp.getOrderedAttr()) {
5638 return emitError() <<
"A worksharing construct that is canceled "
5639 <<
"must not have an ordered clause";
5642 }
else if (cct == ClauseCancellationConstructType::Sections) {
5646 mlir::dyn_cast<SectionsOp>(structuralParent->
getParentOp());
5648 return emitOpError() <<
"cancel sections must appear "
5649 <<
"inside a sections region";
5651 if (sectionsOp.getNowait()) {
5652 return emitError() <<
"A sections construct that is canceled "
5653 <<
"must not have a nowait clause";
5656 if ((cct == ClauseCancellationConstructType::Taskgroup) &&
5657 (!mlir::isa<omp::TaskOp>(structuralParent) &&
5658 !mlir::isa<omp::TaskloopWrapperOp>(structuralParent->
getParentOp()))) {
5659 return emitOpError() <<
"cancel taskgroup must appear "
5660 <<
"inside a task region";
5670 const CancellationPointOperands &clauses) {
5671 CancellationPointOp::build(builder, state, clauses.cancelDirective);
5674LogicalResult CancellationPointOp::verify() {
5675 ClauseCancellationConstructType cct = getCancelDirective();
5678 if (!structuralParent)
5679 return emitOpError() <<
"Orphaned cancellation point";
5681 if ((cct == ClauseCancellationConstructType::Parallel) &&
5682 !mlir::isa<ParallelOp>(structuralParent)) {
5683 return emitOpError() <<
"cancellation point parallel must appear "
5684 <<
"inside a parallel region";
5688 if ((cct == ClauseCancellationConstructType::Loop) &&
5689 !mlir::isa<WsloopOp>(structuralParent->
getParentOp())) {
5690 return emitOpError() <<
"cancellation point loop must appear "
5691 <<
"inside a worksharing-loop region";
5693 if ((cct == ClauseCancellationConstructType::Sections) &&
5694 !mlir::isa<omp::SectionOp>(structuralParent)) {
5695 return emitOpError() <<
"cancellation point sections must appear "
5696 <<
"inside a sections region";
5698 if ((cct == ClauseCancellationConstructType::Taskgroup) &&
5699 (!mlir::isa<omp::TaskOp>(structuralParent) &&
5700 !mlir::isa<omp::TaskloopWrapperOp>(structuralParent->
getParentOp()))) {
5701 return emitOpError() <<
"cancellation point taskgroup must appear "
5702 <<
"inside a task region";
5711LogicalResult MapBoundsOp::verify() {
5712 auto extent = getExtent();
5714 if (!extent && !upperbound)
5715 return emitError(
"expected extent or upperbound.");
5722 PrivateClauseOp::build(
5723 odsBuilder, odsState, symName,
nullptr, type,
5724 DataSharingClauseTypeAttr::get(odsBuilder.
getContext(),
5725 DataSharingClauseType::Private));
5728LogicalResult PrivateClauseOp::verifyRegions() {
5729 Type argType = getArgType();
5730 auto verifyTerminator = [&](
Operation *terminator,
bool yieldsValue,
5731 bool yieldsAnyType) -> LogicalResult {
5735 if (!llvm::isa<YieldOp>(terminator))
5737 <<
"expected exit block terminator to be an `omp.yield` op.";
5739 YieldOp yieldOp = llvm::cast<YieldOp>(terminator);
5740 TypeRange yieldedTypes = yieldOp.getResults().getTypes();
5743 if (yieldedTypes.empty())
5747 <<
"Did not expect any values to be yielded.";
5750 if (yieldedTypes.size() == 1 &&
5751 (yieldsAnyType || yieldedTypes.front() == argType))
5755 <<
"Invalid yielded value. Expected ";
5757 error <<
"a single value";
5759 error <<
"type: " << argType;
5762 if (yieldedTypes.empty())
5765 error << yieldedTypes;
5771 StringRef regionName,
bool yieldsValue,
5772 bool yieldsAnyType =
false) -> LogicalResult {
5773 assert(!region.
empty());
5777 <<
"`" << regionName <<
"`: " <<
"expected " << expectedNumArgs
5780 for (
Block &block : region) {
5794 for (
Region *region : getRegions())
5795 for (
Type ty : region->getArgumentTypes())
5797 return emitError() <<
"Region argument type mismatch: got " << ty
5798 <<
" expected " << argType <<
".";
5801 if (!initRegion.
empty() &&
5806 DataSharingClauseType dsType = getDataSharingType();
5808 if (dsType == DataSharingClauseType::Private && !getCopyRegion().empty())
5809 return emitError(
"`private` clauses do not require a `copy` region.");
5811 if (dsType == DataSharingClauseType::FirstPrivate && getCopyRegion().empty())
5813 "`firstprivate` clauses require at least a `copy` region.");
5815 if (dsType == DataSharingClauseType::FirstPrivate &&
5820 if (dsType == DataSharingClauseType::Linear && !getCopyRegion().empty())
5821 return emitError(
"`linear` clauses do not require a `copy` region.");
5823 if (!getDataAddressRegion().empty()) {
5824 if (dsType != DataSharingClauseType::Linear)
5826 "only `linear` clauses can have a `data_address` region.");
5828 "data_address",
true,
5833 if (!getDeallocRegion().empty() &&
5846 const MaskedOperands &clauses) {
5847 MaskedOp::build(builder, state, clauses.filteredThreadId);
5855 const DispatchOperands &clauses) {
5856 DispatchOp::build(builder, state, clauses.nocontext, clauses.novariants,
5865 const ScanOperands &clauses) {
5866 ScanOp::build(builder, state, clauses.inclusiveVars, clauses.exclusiveVars);
5869LogicalResult ScanOp::verify() {
5870 if (hasExclusiveVars() == hasInclusiveVars())
5872 "Exactly one of EXCLUSIVE or INCLUSIVE clause is expected");
5873 if (WsloopOp parentWsLoopOp = (*this)->getParentOfType<WsloopOp>()) {
5874 if (parentWsLoopOp.hasReductionModifier(ReductionModifier::inscan))
5877 if (SimdOp parentSimdOp = (*this)->getParentOfType<SimdOp>()) {
5878 if (parentSimdOp.hasReductionModifier(ReductionModifier::inscan))
5881 return emitError(
"SCAN directive needs to be enclosed within a parent "
5882 "worksharing loop construct or SIMD construct with INSCAN "
5883 "reduction modifier");
5888 std::optional<uint64_t> alignment) {
5889 if (alignment.has_value()) {
5890 if ((alignment.value() != 0) && !llvm::has_single_bit(alignment.value()))
5892 <<
"ALIGN value : " << alignment.value() <<
" must be power of 2";
5897LogicalResult AllocateDirOp::verify() {
5905LogicalResult AllocSharedMemOp::verify() {
5913LogicalResult FreeSharedMemOp::verify() {
5921LogicalResult WorkdistributeOp::verify() {
5923 return emitOpError() <<
"cannot be a non-innermost combined construct leaf";
5926 Region ®ion = getRegion();
5928 return emitOpError(
"region cannot be empty");
5931 if (entryBlock.
empty())
5932 return emitOpError(
"region must contain a structured block");
5934 bool hasTerminator =
false;
5935 for (
Block &block : region) {
5936 if (isa<TerminatorOp>(block.
back())) {
5937 if (hasTerminator) {
5938 return emitOpError(
"region must have exactly one terminator");
5940 hasTerminator =
true;
5943 if (!hasTerminator) {
5944 return emitOpError(
"region must be terminated with omp.terminator");
5948 if (isa<BarrierOp>(op)) {
5950 "explicit barriers are not allowed in workdistribute region");
5953 if (isa<ParallelOp>(op)) {
5955 "nested parallel constructs not allowed in workdistribute");
5957 if (isa<TeamsOp>(op)) {
5959 "nested teams constructs not allowed in workdistribute");
5963 if (walkResult.wasInterrupted())
5967 if (!llvm::dyn_cast<TeamsOp>(parentOp))
5968 return emitOpError(
"workdistribute must be nested under teams");
5976LogicalResult DeclareSimdOp::verify() {
5979 dyn_cast_if_present<mlir::FunctionOpInterface>((*this)->getParentOp());
5981 return emitOpError() <<
"must be nested inside a function";
5983 if (getInbranch() && getNotinbranch())
5984 return emitOpError(
"cannot have both 'inbranch' and 'notinbranch'");
5990 if (getLinearSyms())
5991 return emitOpError(
"linear privatizers are not allowed");
5997 const DeclareSimdOperands &clauses) {
5999 DeclareSimdOp::build(
6000 odsBuilder, odsState, clauses.alignedVars,
6002 clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
6003 clauses.linearModifiers,
makeArrayAttr(ctx, clauses.linearSyms),
6004 clauses.notinbranch, clauses.simdlen, clauses.uniformVars);
6021 return mlir::failure();
6022 return mlir::success();
6029 for (
unsigned i = 0; i < uniformVars.size(); ++i) {
6032 p << uniformVars[i] <<
" : " << uniformTypes[i];
6047 parser, iterated, iteratedTypes, affinityVars, affinityVarTypes,
6048 [&]() -> ParseResult {
return success(); })))
6082 OpAsmParser::Argument &arg = ivArgs.emplace_back();
6083 if (parser.parseArgument(arg))
6087 if (succeeded(parser.parseOptionalColon())) {
6088 if (parser.parseType(arg.type))
6091 arg.type = parser.getBuilder().getIndexType();
6103 OpAsmParser::UnresolvedOperand lb, ub, st;
6104 if (parser.parseOperand(lb) || parser.parseKeyword(
"to") ||
6105 parser.parseOperand(ub) || parser.parseKeyword(
"step") ||
6106 parser.parseOperand(st))
6111 steps.push_back(st);
6119 if (ivArgs.size() != lbs.size())
6121 <<
"mismatch: " << ivArgs.size() <<
" variables but " << lbs.size()
6124 for (
auto &arg : ivArgs) {
6125 lbTypes.push_back(arg.type);
6126 ubTypes.push_back(arg.type);
6127 stepTypes.push_back(arg.type);
6147 for (
unsigned i = 0, e = lbs.size(); i < e; ++i) {
6150 p << lbs[i] <<
" to " << ubs[i] <<
" step " << steps[i];
6158YieldOp IteratorOp::getYieldOp() {
6160 for (
Block &block : getRegion()) {
6162 dyn_cast_if_present<YieldOp>(block.
empty() ?
nullptr : &block.
back());
6172LogicalResult IteratorOp::verify() {
6173 auto iteratedTy = llvm::dyn_cast<omp::IteratedType>(getIterated().
getType());
6175 return emitOpError() <<
"result must be omp.iterated<entry_ty>";
6178 if (getLoopLowerBounds().size() != ivs.size())
6179 return emitOpError() <<
"number of range arguments and IVs do not match";
6181 for (
auto [lb, iv] : llvm::zip_equal(getLoopLowerBounds(), ivs)) {
6182 if (lb.getType() != iv.getType())
6183 return emitOpError()
6184 <<
"range argument type does not match corresponding IV type";
6187 for (
Value step : getLoopSteps()) {
6189 return emitOpError() <<
"loop step must not be zero";
6192 YieldOp yield = getYieldOp();
6194 bool hasYield = llvm::any_of(getRegion(), [](
Block &block) {
6195 return !block.
empty() && isa<YieldOp>(block.
back());
6198 return emitOpError() <<
"region must contain exactly one omp.yield";
6199 return emitOpError() <<
"region must be terminated by omp.yield";
6202 if (yield.getNumOperands() != 1)
6203 return emitOpError()
6204 <<
"omp.yield in omp.iterator region must yield exactly one value";
6206 mlir::Type yieldedTy = yield.getOperand(0).getType();
6207 mlir::Type elemTy = iteratedTy.getElementType();
6209 if (yieldedTy != elemTy)
6210 return emitOpError() <<
"omp.iterated element type (" << elemTy
6211 <<
") does not match omp.yield operand type ("
6212 << yieldedTy <<
")";
6217LogicalResult IteratorOp::verifyRegions() {
6219 for (
Block &block : getRegion()) {
6223 <<
"expected exit block terminator to be an `omp.yield` op.";
6236 return emitOpError() <<
"expected symbol reference '" << getSymName()
6237 <<
"' to point to a global variable";
6239 if (isa<FunctionOpInterface>(symbol))
6240 return emitOpError() <<
"expected symbol reference '" << getSymName()
6241 <<
"' to point to a global variable, not a function";
6246#define GET_ATTRDEF_CLASSES
6247#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
6249#define GET_OP_CLASSES
6250#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
6252#define GET_TYPEDEF_CLASSES
6253#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
if(failed(verifyVectorMemoryOp(getOperation(), memrefType, getVectorType()))) return failure()
static std::optional< int64_t > getUpperBound(Value iv)
Gets the constant upper bound on an affine.for iv.
static LogicalResult verifyRegion(emitc::SwitchOp op, Region ®ion, const Twine &name)
static const mlir::GenInfo * generator
static LogicalResult verifyNontemporalClause(Operation *op, OperandRange nontemporalVars)
static DenseI64ArrayAttr makeDenseI64ArrayAttr(MLIRContext *ctx, const ArrayRef< int64_t > intArray)
static void printDependVarList(OpAsmPrinter &p, Operation *op, OperandRange dependVars, TypeRange dependTypes, std::optional< ArrayAttr > dependKinds, OperandRange iteratedVars, TypeRange iteratedTypes, std::optional< ArrayAttr > iteratedKinds)
Print Depend clause.
static ParseResult parseTargetOpRegion(OpAsmParser &parser, Region ®ion, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &hasDeviceAddrVars, SmallVectorImpl< Type > &hasDeviceAddrTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &hostEvalVars, SmallVectorImpl< Type > &hostEvalTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &mapVars, SmallVectorImpl< Type > &mapTypes, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier, DenseI64ArrayAttr &privateMaps)
static constexpr StringRef getPrivateNeedsBarrierSpelling()
static void printHeapAllocClause(OpAsmPrinter &p, Operation *op, TypeAttr inType, ValueRange typeparams, TypeRange typeparamsTypes, ValueRange shape, TypeRange shapeTypes)
static ParseResult parseInReductionPrivateReductionRegion(OpAsmParser &parser, Region ®ion, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &inReductionVars, SmallVectorImpl< Type > &inReductionTypes, DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &reductionVars, SmallVectorImpl< Type > &reductionTypes, DenseBoolArrayAttr &reductionByref, ArrayAttr &reductionSyms)
static ParseResult parseInReductionPrivateRegion(OpAsmParser &parser, Region ®ion, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &inReductionVars, SmallVectorImpl< Type > &inReductionTypes, DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier)
static ArrayAttr makeArrayAttr(MLIRContext *context, llvm::ArrayRef< Attribute > attrs)
static ParseResult parseClauseAttr(AsmParser &parser, ClauseAttr &attr)
static void printLinearPrivateReductionRegion(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange linearVars, TypeRange linearTypes, ValueRange linearStepVars, TypeRange linearStepTypes, ArrayAttr linearVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier, ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes, DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms)
static ParseResult parseLinearPrivateReductionRegion(OpAsmParser &parser, Region ®ion, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &linearVars, SmallVectorImpl< Type > &linearTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &linearStepVars, SmallVectorImpl< Type > &linearStepTypes, ArrayAttr &linearVarTypes, ArrayAttr &linearModifiers, ArrayAttr &linearSyms, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &reductionVars, SmallVectorImpl< Type > &reductionTypes, DenseBoolArrayAttr &reductionByref, ArrayAttr &reductionSyms)
static void printDynGroupprivateClause(OpAsmPrinter &printer, Operation *op, AccessGroupModifierAttr modifierFirst, FallbackModifierAttr modifierSecond, Value dynGroupprivateSize, Type sizeType)
static void printAllocateAndAllocator(OpAsmPrinter &p, Operation *op, OperandRange allocateVars, TypeRange allocateTypes, OperandRange allocatorVars, TypeRange allocatorTypes)
Print allocate clause.
static DenseBoolArrayAttr makeDenseBoolArrayAttr(MLIRContext *ctx, const ArrayRef< bool > boolArray)
static std::string generateLoopNestingName(StringRef prefix, CanonicalLoopOp op)
Generate a name of a canonical loop nest of the format <prefix>(_r<idx>_s<idx>)*.
static ParseResult parseAffinityClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &iterated, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &affinityVars, SmallVectorImpl< Type > &iteratedTypes, SmallVectorImpl< Type > &affinityVarTypes)
static void printSplitIteratedList(OpAsmPrinter &p, ValueRange iteratedVars, TypeRange iteratedTypes, ValueRange plainVars, TypeRange plainTypes, PrintPrefixFn &&printPrefixForPlain, PrintPrefixFn &&printPrefixForIterated)
static LogicalResult verifyDependVarList(Operation *op, std::optional< ArrayAttr > dependKinds, OperandRange dependVars, std::optional< ArrayAttr > iteratedKinds, OperandRange iteratedVars)
Verifies Depend clause.
static void addUnknownMemoryEffects(SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Add arbitrary read, write, allocate, and free effects.
static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName, ValueRange argsSubrange, std::optional< MapPrintArgs > mapArgs)
static void printAffinityClause(OpAsmPrinter &p, Operation *op, ValueRange iterated, ValueRange affinityVars, TypeRange iteratedTypes, TypeRange affinityVarTypes)
static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region ®ion, const AllRegionPrintArgs &args)
static ParseResult parseGranularityClause(OpAsmParser &parser, ClauseTypeAttr &prescriptiveness, std::optional< OpAsmParser::UnresolvedOperand > &operand, Type &operandType, std::optional< ClauseType >(*symbolizeClause)(StringRef), StringRef clauseName)
static void printIteratorHeader(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange lbs, ValueRange ubs, ValueRange steps, TypeRange, TypeRange, TypeRange)
static LogicalResult verifyDeclareTargetAttr(Operation *op, Attribute attr)
static ParseResult parseHeapAllocClause(OpAsmParser &parser, TypeAttr &inTypeAttr, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &typeparams, SmallVectorImpl< Type > &typeparamsTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &shape, SmallVectorImpl< Type > &shapeTypes)
operation ::= $in_type ( ( $typeparams ) )? ( , $shape )?
static void printInReductionClause(OpAsmPrinter &p, Operation *op, ValueRange inReductionVars, TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref, ArrayAttr inReductionSyms)
Prints an in_reduction clause for an operation that does not give its list items entry block argument...
static void printClauseWithRegionArgs(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName, ValueRange argsSubrange, ValueRange operands, TypeRange types, ArrayAttr symbols=nullptr, DenseI64ArrayAttr mapIndices=nullptr, DenseBoolArrayAttr byref=nullptr, ArrayAttr modifier=nullptr, UnitAttr needsBarrier=nullptr)
static ParseResult parseIteratorHeader(OpAsmParser &parser, Region ®ion, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &lbs, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &ubs, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &steps, SmallVectorImpl< Type > &lbTypes, SmallVectorImpl< Type > &ubTypes, SmallVectorImpl< Type > &stepTypes)
static LogicalResult verifyReductionVarList(Operation *op, std::optional< ArrayAttr > reductionSyms, OperandRange reductionVars, std::optional< ArrayRef< bool > > reductionByref, ArrayAttr reductionMods=nullptr)
Verifies Reduction Clause.
static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region ®ion, AllRegionParseArgs args)
static ParseResult parseLoopTransformClis(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &generateesOperands, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &applyeesOperands)
static ParseResult parseSynchronizationHint(OpAsmParser &parser, IntegerAttr &hintAttr)
Parses a Synchronization Hint clause.
static void printLinearClause(OpAsmPrinter &p, Operation *op, ValueRange linearVars, TypeRange linearTypes, ValueRange linearStepVars, TypeRange stepVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms)
Print Linear Clause.
static void printScheduleClause(OpAsmPrinter &p, Operation *op, ClauseScheduleKindAttr scheduleKind, ScheduleModifierAttr scheduleMod, UnitAttr scheduleSimd, Value scheduleChunk, Type scheduleChunkType)
Print schedule clause.
static ParseResult parsePrivateReductionRegion(OpAsmParser &parser, Region ®ion, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &reductionVars, SmallVectorImpl< Type > &reductionTypes, DenseBoolArrayAttr &reductionByref, ArrayAttr &reductionSyms)
static void printCopyprivate(OpAsmPrinter &p, Operation *op, OperandRange copyprivateVars, TypeRange copyprivateTypes, std::optional< ArrayAttr > copyprivateSyms)
Print Copyprivate clause.
static ParseResult parseOrderClause(OpAsmParser &parser, ClauseOrderKindAttr &order, OrderModifierAttr &orderMod)
static bool mapTypeToBool(ClauseMapFlags value, ClauseMapFlags flag)
static void printAlignedClause(OpAsmPrinter &p, Operation *op, ValueRange alignedVars, TypeRange alignedTypes, std::optional< ArrayAttr > alignments)
Print Aligned Clause.
static bool targetInReductionCapturedBy(Value inReductionVar, Value mapVarPtr)
An omp.target in_reduction operand is captured by a map_entries entry when the entry's MapInfoOp var_...
static LogicalResult verifySynchronizationHint(Operation *op, uint64_t hint)
Verifies a synchronization hint clause.
static ParseResult parseUseDeviceAddrUseDevicePtrRegion(OpAsmParser &parser, Region ®ion, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &useDeviceAddrVars, SmallVectorImpl< Type > &useDeviceAddrTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &useDevicePtrVars, SmallVectorImpl< Type > &useDevicePtrTypes)
static void printPrivateReductionRegion(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier, ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes, DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms)
static ParseResult parseUniformClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &uniformVars, SmallVectorImpl< Type > &uniformTypes)
uniform ::= uniform ( uniform-list ) uniform-list := uniform-val (, uniform-val)* uniform-val := ssa-...
static void printInReductionPrivateRegion(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange inReductionVars, TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref, ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier)
static LogicalResult verifyAllocateClause(Operation *op, ValueRange allocateVars, ValueRange allocatorVars, DenseI64ArrayAttr allocateAlignments, DenseI64ArrayAttr allocatePrivateIndices, ValueRange privateVars={}, ArrayAttr privateSyms=nullptr, bool requirePrivateIndices=false)
static void printSynchronizationHint(OpAsmPrinter &p, Operation *op, IntegerAttr hintAttr)
Prints a Synchronization Hint clause.
static void printGranularityClause(OpAsmPrinter &p, Operation *op, ClauseTypeAttr prescriptiveness, Value operand, mlir::Type operandType, StringRef(*stringifyClauseType)(ClauseType))
static ParseResult parseDependVarList(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &dependVars, SmallVectorImpl< Type > &dependTypes, ArrayAttr &dependKinds, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &iteratedVars, SmallVectorImpl< Type > &iteratedTypes, ArrayAttr &iteratedKinds)
depend-entry-list ::= depend-entry | depend-entry-list , depend-entry depend-entry ::= depend-kind ->...
static void getTaskReductionEffects(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
static Operation * getParentInSameDialect(Operation *thisOp)
static void printUniformClause(OpAsmPrinter &p, Operation *op, ValueRange uniformVars, TypeRange uniformTypes)
Print Uniform Clauses.
static LogicalResult verifyCopyprivateVarList(Operation *op, OperandRange copyprivateVars, std::optional< ArrayAttr > copyprivateSyms)
Verifies CopyPrivate Clause.
static LogicalResult verifyAlignedClause(Operation *op, std::optional< ArrayAttr > alignments, OperandRange alignedVars)
static ParseResult parsePrivateRegion(OpAsmParser &parser, Region ®ion, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier)
static void printNumTasksClause(OpAsmPrinter &p, Operation *op, ClauseNumTasksTypeAttr numTasksMod, Value numTasks, mlir::Type numTasksType)
static void printLoopTransformClis(OpAsmPrinter &p, TileOp op, OperandRange generatees, OperandRange applyees)
static ParseResult parseDynGroupprivateClause(OpAsmParser &parser, AccessGroupModifierAttr &accessGroupAttr, FallbackModifierAttr &fallbackAttr, std::optional< OpAsmParser::UnresolvedOperand > &dynGroupprivateSize, Type &sizeType)
static void printPrivateRegion(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier)
static ParseResult parseSplitIteratedList(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &iteratedVars, SmallVectorImpl< Type > &iteratedTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &plainVars, SmallVectorImpl< Type > &plainTypes, ParsePrefixFn &&parsePrefix)
static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange taskReductionVars, TypeRange taskReductionTypes, DenseBoolArrayAttr taskReductionByref, ArrayAttr taskReductionSyms)
static LogicalResult verifyMapInfoForMapClause(Operation *op, mlir::omp::MapInfoOp mapInfoOp, llvm::DenseSet< mlir::TypedValue< mlir::omp::PointerLikeType > > &updateToVars, llvm::DenseSet< mlir::TypedValue< mlir::omp::PointerLikeType > > &updateFromVars)
static void getPrivatisationEffects(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Privatisation may read the original as a mold or firstprivate copy source. Any lastprivate copy-back ...
static void printInReductionPrivateReductionRegion(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange inReductionVars, TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref, ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier, ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes, DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms)
static void getBarrierEffects(llvm::SmallVectorImpl< SideEffects::EffectInstance< MemoryEffects::Effect > > &effects)
static LogicalResult verifyOrderedParent(Operation &op)
static void printOrderClause(OpAsmPrinter &p, Operation *op, ClauseOrderKindAttr order, OrderModifierAttr orderMod)
static ParseResult parseBlockArgClause(OpAsmParser &parser, llvm::SmallVectorImpl< OpAsmParser::Argument > &entryBlockArgs, StringRef keyword, std::optional< MapParseArgs > mapArgs)
static LogicalResult verifyPrivateVarsMapping(TargetOp targetOp)
static ParseResult parseScheduleClause(OpAsmParser &parser, ClauseScheduleKindAttr &scheduleAttr, ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd, std::optional< OpAsmParser::UnresolvedOperand > &chunkSize, Type &chunkType)
schedule ::= schedule ( sched-list ) sched-list ::= sched-val | sched-val sched-list | sched-val ,...
static LogicalResult verifyDynGroupprivateClause(Operation *op, AccessGroupModifierAttr accessGroup, FallbackModifierAttr fallback, Value dynGroupprivateSize)
static LogicalResult verifyLinearModifiers(Operation *op, std::optional< ArrayAttr > linearModifiers, OperandRange linearVars, bool isDeclareSimd=false)
OpenMP 5.2, Section 5.4.6: "A linear-modifier may be specified as ref or uval only on a declare simd ...
static void printClauseAttr(OpAsmPrinter &p, Operation *op, ClauseAttr attr)
static LogicalResult verifyLinearSyms(OpType op)
Verifies that each linear variable has a linear privatizer.
static ParseResult parseAllocateAndAllocator(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &allocateVars, SmallVectorImpl< Type > &allocateTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &allocatorVars, SmallVectorImpl< Type > &allocatorTypes)
Parse an allocate clause with allocators and a list of operands with types.
static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op, ArrayAttr membersIdx)
static void getReductionEffects(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Reductions initialise private accumulators and combine them into the original variables....
static ParseResult parseClauseWithRegionArgs(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &operands, SmallVectorImpl< Type > &types, SmallVectorImpl< OpAsmParser::Argument > ®ionPrivateArgs, ArrayAttr *symbols=nullptr, DenseI64ArrayAttr *mapIndices=nullptr, DenseBoolArrayAttr *byref=nullptr, ArrayAttr *modifier=nullptr, UnitAttr *needsBarrier=nullptr)
static void printCaptureType(OpAsmPrinter &p, Operation *op, VariableCaptureKindAttr mapCaptureType)
static LogicalResult verifyReductionModifiers(Operation *op, OperandRange reductionVars, ArrayAttr reductionMods)
Verify the per-variable modifier list has one entry per reduction variable.
static void getTaskSchedulingPointEffects(llvm::SmallVectorImpl< SideEffects::EffectInstance< MemoryEffects::Effect > > &effects)
static LogicalResult verifyNumTeamsClause(Operation *op, Value numTeamsLower, OperandRange numTeamsUpperVars)
static bool opInGlobalImplicitParallelRegion(Operation *op)
static void printTargetOpRegion(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange hasDeviceAddrVars, TypeRange hasDeviceAddrTypes, ValueRange hostEvalVars, TypeRange hostEvalTypes, ValueRange mapVars, TypeRange mapTypes, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier, DenseI64ArrayAttr privateMaps)
static void printUseDeviceAddrUseDevicePtrRegion(OpAsmPrinter &p, Operation *op, Region ®ion, ValueRange useDeviceAddrVars, TypeRange useDeviceAddrTypes, ValueRange useDevicePtrVars, TypeRange useDevicePtrTypes)
static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars, OperandRange mapIterated)
static LogicalResult verifyPrivateVarList(OpType &op)
static ParseResult parseNumTasksClause(OpAsmParser &parser, ClauseNumTasksTypeAttr &numTasksMod, std::optional< OpAsmParser::UnresolvedOperand > &numTasks, Type &numTasksType)
LogicalResult verifyAlignment(Operation &op, std::optional< uint64_t > alignment)
Verifies align clause in allocate directive.
static ParseResult parseAlignedClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &alignedVars, SmallVectorImpl< Type > &alignedTypes, ArrayAttr &alignmentsAttr)
aligned ::= aligned ( aligned-list ) aligned-list := aligned-val | aligned-val aligned-list aligned-v...
static ParseResult parseLinearClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &linearVars, SmallVectorImpl< Type > &linearTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &linearStepVars, SmallVectorImpl< Type > &linearStepTypes, ArrayAttr &linearModifiers, ArrayAttr &linearSyms)
linear ::= linear ( linear-list ) linear-list := linear-val | linear-val linear-list linear-val := ss...
static LogicalResult checkApplyeesNesting(TileOp op)
Check properties of the loop nest consisting of the transformation's applyees:
static ParseResult parseCaptureType(OpAsmParser &parser, VariableCaptureKindAttr &mapCaptureType)
static ParseResult parseTaskReductionRegion(OpAsmParser &parser, Region ®ion, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &taskReductionVars, SmallVectorImpl< Type > &taskReductionTypes, DenseBoolArrayAttr &taskReductionByref, ArrayAttr &taskReductionSyms)
static ParseResult parseGrainsizeClause(OpAsmParser &parser, ClauseGrainsizeTypeAttr &grainsizeMod, std::optional< OpAsmParser::UnresolvedOperand > &grainsize, Type &grainsizeType)
static ParseResult parseCopyprivate(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > ©privateVars, SmallVectorImpl< Type > ©privateTypes, ArrayAttr ©privateSyms)
copyprivate-entry-list ::= copyprivate-entry | copyprivate-entry-list , copyprivate-entry copyprivate...
static ParseResult parseInReductionClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &inReductionVars, SmallVectorImpl< Type > &inReductionTypes, DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms)
Parses an in_reduction clause for an operation that does not give its list items entry block argument...
static LogicalResult verifyMapInfoDefinedArgs(Operation *op, StringRef clauseName, OperandRange vars)
static void printGrainsizeClause(OpAsmPrinter &p, Operation *op, ClauseGrainsizeTypeAttr grainsizeMod, Value grainsize, mlir::Type grainsizeType)
static ParseResult verifyScheduleModifiers(OpAsmParser &parser, SmallVectorImpl< SmallString< 12 > > &modifiers)
static Type getElementType(Type type, ArrayRef< int32_t > indices, function_ref< InFlightDiagnostic(StringRef)> emitErrorFn)
Walks the given type hierarchy with the given indices, potentially down to component granularity,...
static bool isUnique(It begin, It end)
static LogicalResult emit(SolverOp solver, const SMTEmissionOptions &options, mlir::raw_indented_ostream &stream)
Emit the SMT operations in the given 'solver' to the 'stream'.
static SmallVector< Value > getTileSizes(Location loc, x86::amx::TileType tType, RewriterBase &rewriter)
Maps the 2-dim vector shape to the two 16-bit tile sizes.
This base class exposes generic asm parser hooks, usable across the various derived parsers.
virtual ParseResult parseMinus()=0
Parse a '-' token.
@ Paren
Parens surrounding zero or more operands.
@ None
Zero or more operands with no delimiters.
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalEqual()=0
Parse a = token if present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
virtual ParseResult parseRParen()=0
Parse a ) token.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
virtual ParseResult parseOptionalColon()=0
Parse a : token if present.
virtual ParseResult parseLSquare()=0
Parse a [ token.
virtual ParseResult parseRSquare()=0
Parse a ] token.
ParseResult parseInteger(IntT &result)
Parse an integer value from the stream.
virtual ParseResult parseOptionalArrow()=0
Parse a '->' token if present.
virtual ParseResult parseLess()=0
Parse a '<' token.
virtual ParseResult parseEqual()=0
Parse a = token.
virtual ParseResult parseColonType(Type &result)=0
Parse a colon followed by a type.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual ParseResult parseOptionalComma()=0
Parse a , token if present.
virtual ParseResult parseColon()=0
Parse a : token.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseOptionalLess()=0
Parse a '<' token if present.
virtual ParseResult parseArrow()=0
Parse a '->' token.
virtual ParseResult parseLParen()=0
Parse a ( token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
virtual ParseResult parseComma()=0
Parse a , token.
virtual ParseResult parseOptionalLParen()=0
Parse a ( token if present.
ParseResult parseKeyword(StringRef keyword)
Parse a given keyword.
virtual ParseResult parseOptionalLSquare()=0
Parse a [ token if present.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
Attributes are known-constant values of operations.
This class represents an argument of a Block.
Block represents an ordered list of Operations.
MutableArrayRef< BlockArgument > BlockArgListType
ValueTypeRange< BlockArgListType > getArgumentTypes()
Return a range containing the types of the arguments for this block.
unsigned getNumSuccessors()
BlockArgument getArgument(unsigned i)
unsigned getNumArguments()
SuccessorRange getSuccessors()
Operation * getTerminator()
Get the terminator operation of this block.
bool mightHaveTerminator()
Return "true" if this block might have a terminator.
BlockArgListType getArguments()
IntegerAttr getI64IntegerAttr(int64_t value)
IntegerType getIntegerType(unsigned width)
MLIRContext * getContext() const
Attr getAttr(Args &&...args)
Get or construct an instance of the attribute Attr with provided arguments.
Diagnostic & append(Arg1 &&arg1, Arg2 &&arg2, Args &&...args)
Append arguments to the diagnostic.
Diagnostic & appendOp(Operation &op, const OpPrintingFlags &flags)
Append an operation with the given printing flags.
A class for computing basic dominance information.
bool dominates(Operation *a, Operation *b) const
Return true if operation A dominates operation B, i.e.
This class represents a diagnostic that is inflight and set to be reported.
Diagnostic & attachNote(std::optional< Location > noteLoc=std::nullopt)
Attaches a note to this diagnostic.
MLIRContext is the top-level object for a collection of MLIR operations.
NamedAttribute represents a combination of a name and an Attribute value.
StringAttr getName() const
Return the name of the attribute.
Attribute getValue() const
Return the value of the attribute.
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult parseRegion(Region ®ion, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region.
virtual ParseResult parseArgument(Argument &result, bool allowType=false, bool allowAttrs=false)=0
Parse a single argument with the following syntax:
virtual ParseResult parseArgumentList(SmallVectorImpl< Argument > &result, Delimiter delimiter=Delimiter::None, bool allowType=false, bool allowAttrs=false)=0
Parse zero or more arguments with a specified surrounding delimiter.
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
ParseResult resolveOperands(Operands &&operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
virtual ParseResult parseOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None, bool allowResultNumber=true, int requiredOperandCount=-1)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
virtual void printRegionArgument(BlockArgument arg, ArrayRef< NamedAttribute > argAttrs={}, bool omitType=false)=0
Print a block argument in the usual format of: ssaName : type {attr1=42} loc("here") where location p...
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
This class helps build Operations.
This class represents an operand of an operation.
Set of flags used to control the behavior of the various IR print methods (e.g.
This class provides the API for ops that are known to be isolated from above.
This class provides the API for ops that are known to be terminators.
This class indicates that the regions associated with this op don't have terminators.
This class implements the operand iterators for the Operation class.
type_range getType() const
Operation is the basic unit of execution within MLIR.
Dialect * getDialect()
Return the dialect this operation is associated with, or nullptr if the associated dialect is not loa...
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Block * getBlock()
Returns the operation block that contains this operation.
unsigned getNumRegions()
Returns the number of regions held by this operation.
Location getLoc()
The source location the operation was defined or derived from.
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
operand_range getOperands()
Returns an iterator on the underlying Value's.
user_range getUsers()
Returns a range of all users.
Region * getParentRegion()
Returns the region to which the instruction belongs.
MLIRContext * getContext()
Return the context this operation is associated with.
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
This class contains a list of basic blocks and a link to the parent operation it is attached to.
BlockArgListType getArguments()
OpIterator op_begin()
Return iterators that walk the operations nested directly within this region.
bool isAncestor(Region *other)
Return true if this region is ancestor of the other region.
iterator_range< OpIterator > getOps()
unsigned getNumArguments()
Location getLoc()
Return a location for this region.
BlockArgument getArgument(unsigned i)
Operation * getParentOp()
Return the parent operation this region is attached to.
BlockListType & getBlocks()
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replace the results of the given (original) op with a new op that is created without verification (re...
This class represents a specific instance of an effect.
static DerivedEffect * get()
This class represents a collection of SymbolTables.
virtual Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
static Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
This class provides an abstraction over the various different ranges of value types.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class provides an abstraction over the different types of ranges over Values.
type_range getType() const
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
MLIRContext * getContext() const
Utility to get the associated MLIRContext that this value is defined in.
Type getType() const
Return the type of this value.
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
A utility result that is used to signal how to proceed with an ongoing walk:
static WalkResult advance()
static DenseArrayAttrImpl get(MLIRContext *context, ArrayRef< bool > content)
ArrayRef< T > asArrayRef() const
bool isReachableFromEntry(Block *a) const
Return true if the specified block is reachable from the entry block of its region.
Operation * getOwner() const
Return the owner of this operand.
TargetEnterDataOperands TargetEnterExitUpdateDataOperands
omp.target_enter_data, omp.target_exit_data and omp.target_update take the same clauses,...
std::tuple< NewCliOp, OpOperand *, OpOperand * > decodeCli(mlir::Value cli)
Find the omp.new_cli, generator, and consumer of a canonical loop info.
ClauseProcBindKind convertProcBindKind(llvm::omp::ProcBindKind kind)
Convert a proc_bind kind from the LLVM frontend enum to the corresponding OpenMP dialect enum.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
detail::DenseArrayAttrImpl< int64_t > DenseI64ArrayAttr
function_ref< void(Value, StringRef)> OpAsmSetValueNameFn
A functor used to set the name of the start of a result group of an operation.
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
llvm::DenseSet< ValueT, ValueInfoT > DenseSet
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
bool isPure(Operation *op)
Returns true if the given operation is pure, i.e., is speculatable that does not touch memory.
detail::constant_int_predicate_matcher m_Zero()
Matches a constant scalar / vector splat / tensor splat integer zero.
std::conditional_t< std::is_same_v< Ty, mlir::Type >, mlir::Value, detail::TypedValue< Ty > > TypedValue
If Ty is mlir::Type this will select Value instead of having a wrapper around it.
llvm::TypeSwitch< T, ResultT > TypeSwitch
SmallVector< Loops, 8 > tile(ArrayRef< scf::ForOp > forOps, ArrayRef< Value > sizes, ArrayRef< scf::ForOp > targets)
Performs tiling fo imperfectly nested loops (with interchange) by strip-mining the forOps by sizes an...
detail::DenseArrayAttrImpl< bool > DenseBoolArrayAttr
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
function_ref< void(Block *, StringRef)> OpAsmSetBlockNameFn
A functor used to set the name of blocks in regions directly nested under an operation.
This is the representation of an operand reference.
This class provides APIs and verifiers for ops with regions having a single block.
This represents an operation in an abstracted form, suitable for use with the builder APIs.
T & getOrAddProperties()
Get (or create) the properties of the provided type to be set on the operation on creation.
void addOperands(ValueRange newOperands)
void addAttributes(ArrayRef< NamedAttribute > newAttributes)
Add an array of named attributes.
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
void addTypes(ArrayRef< Type > newTypes)
Region * addRegion()
Create a region that should be attached to the operation.
Extended TargetOperands with kernel_type attribute.
TargetExecModeAttr kernelType
Kernel execution mode for the target region.