MLIR 24.0.0git
OpenMPDialect.cpp
Go to the documentation of this file.
1//===- OpenMPDialect.cpp - MLIR Dialect for OpenMP implementation ---------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file implements the OpenMP dialect and its operations.
10//
11//===----------------------------------------------------------------------===//
12
18#include "mlir/IR/Attributes.h"
20#include "mlir/IR/Matchers.h"
23#include "mlir/IR/SymbolTable.h"
26
27#include "llvm/ADT/ArrayRef.h"
28#include "llvm/ADT/PostOrderIterator.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/STLForwardCompat.h"
31#include "llvm/ADT/SmallString.h"
32#include "llvm/ADT/StringRef.h"
33#include "llvm/ADT/TypeSwitch.h"
34#include "llvm/ADT/bit.h"
35#include "llvm/Support/InterleavedRange.h"
36#include <cstddef>
37#include <iterator>
38#include <optional>
39#include <variant>
40
41#include "mlir/Dialect/OpenMP/OpenMPOpsDialect.cpp.inc"
42#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.cpp.inc"
43#include "mlir/Dialect/OpenMP/OpenMPOpsInterfaces.cpp.inc"
44#include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.cpp.inc"
45
46using namespace mlir;
47using namespace mlir::omp;
48
51 return attrs.empty() ? nullptr : ArrayAttr::get(context, attrs);
52}
53
56 return boolArray.empty() ? nullptr : DenseBoolArrayAttr::get(ctx, boolArray);
57}
58
61 return intArray.empty() ? nullptr : DenseI64ArrayAttr::get(ctx, intArray);
62}
63
64namespace {
65struct MemRefPointerLikeModel
66 : public PointerLikeType::ExternalModel<MemRefPointerLikeModel,
67 MemRefType> {
68 Type getElementType(Type pointer) const {
69 return llvm::cast<MemRefType>(pointer).getElementType();
70 }
71};
72
73struct LLVMPointerPointerLikeModel
74 : public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,
75 LLVM::LLVMPointerType> {
76 Type getElementType(Type pointer) const { return Type(); }
77};
78} // namespace
79
80/// Generate a name of a canonical loop nest of the format
81/// `<prefix>(_r<idx>_s<idx>)*`. Hereby, `_r<idx>` identifies the region
82/// argument index of an operation that has multiple regions, if the operation
83/// has multiple regions.
84/// `_s<idx>` identifies the position of an operation within a region, where
85/// only operations that may potentially contain loops ("container operations"
86/// i.e. have region arguments) are counted. Again, it is omitted if there is
87/// only one such operation in a region. If there are canonical loops nested
88/// inside each other, also may also use the format `_d<num>` where <num> is the
89/// nesting depth of the loop.
90///
91/// The generated name is a best-effort to make canonical loop unique within an
92/// SSA namespace. This also means that regions with IsolatedFromAbove property
93/// do not consider any parents or siblings.
94static std::string generateLoopNestingName(StringRef prefix,
95 CanonicalLoopOp op) {
96 struct Component {
97 /// If true, this component describes a region operand of an operation (the
98 /// operand's owner) If false, this component describes an operation located
99 /// in a parent region
100 bool isRegionArgOfOp;
101 bool skip = false;
102 bool isUnique = false;
103
104 size_t idx;
105 Operation *op;
106 Region *parentRegion;
107 size_t loopDepth;
108
109 Operation *&getOwnerOp() {
110 assert(isRegionArgOfOp && "Must describe a region operand");
111 return op;
112 }
113 size_t &getArgIdx() {
114 assert(isRegionArgOfOp && "Must describe a region operand");
115 return idx;
116 }
117
118 Operation *&getContainerOp() {
119 assert(!isRegionArgOfOp && "Must describe a operation of a region");
120 return op;
121 }
122 size_t &getOpPos() {
123 assert(!isRegionArgOfOp && "Must describe a operation of a region");
124 return idx;
125 }
126 bool isLoopOp() const {
127 assert(!isRegionArgOfOp && "Must describe a operation of a region");
128 return isa<CanonicalLoopOp>(op);
129 }
130 Region *&getParentRegion() {
131 assert(!isRegionArgOfOp && "Must describe a operation of a region");
132 return parentRegion;
133 }
134 size_t &getLoopDepth() {
135 assert(!isRegionArgOfOp && "Must describe a operation of a region");
136 return loopDepth;
137 }
138
139 void skipIf(bool v = true) { skip = skip || v; }
140 };
141
142 // List of ancestors, from inner to outer.
143 // Alternates between
144 // * region argument of an operation
145 // * operation within a region
146 SmallVector<Component> components;
147
148 // Gather a list of parent regions and operations, and the position within
149 // their parent
150 Operation *o = op.getOperation();
151 while (o) {
152 // Operation within a region
153 Region *r = o->getParentRegion();
154 if (!r)
155 break;
156
157 llvm::ReversePostOrderTraversal<Block *> traversal(&r->getBlocks().front());
158 size_t idx = 0;
159 bool found = false;
160 size_t sequentialIdx = -1;
161 bool isOnlyContainerOp = true;
162 for (Block *b : traversal) {
163 for (Operation &op : *b) {
164 if (&op == o && !found) {
165 sequentialIdx = idx;
166 found = true;
167 }
168 if (op.getNumRegions()) {
169 idx += 1;
170 if (idx > 1)
171 isOnlyContainerOp = false;
172 }
173 if (found && !isOnlyContainerOp)
174 break;
175 }
176 }
177
178 Component &containerOpInRegion = components.emplace_back();
179 containerOpInRegion.isRegionArgOfOp = false;
180 containerOpInRegion.isUnique = isOnlyContainerOp;
181 containerOpInRegion.getContainerOp() = o;
182 containerOpInRegion.getOpPos() = sequentialIdx;
183 containerOpInRegion.getParentRegion() = r;
184
185 Operation *parent = r->getParentOp();
186
187 // Region argument of an operation
188 Component &regionArgOfOperation = components.emplace_back();
189 regionArgOfOperation.isRegionArgOfOp = true;
190 regionArgOfOperation.isUnique = true;
191 regionArgOfOperation.getArgIdx() = 0;
192 regionArgOfOperation.getOwnerOp() = parent;
193
194 // The IsolatedFromAbove trait of the parent operation implies that each
195 // individual region argument has its own separate namespace, so no
196 // ambiguity.
197 if (!parent || parent->hasTrait<mlir::OpTrait::IsIsolatedFromAbove>())
198 break;
199
200 // Component only needed if operation has multiple region operands. Region
201 // arguments may be optional, but we currently do not consider this.
202 if (parent->getRegions().size() > 1) {
203 auto getRegionIndex = [](Operation *o, Region *r) {
204 for (auto [idx, region] : llvm::enumerate(o->getRegions())) {
205 if (&region == r)
206 return idx;
207 }
208 llvm_unreachable("Region not child of its parent operation");
209 };
210 regionArgOfOperation.isUnique = false;
211 regionArgOfOperation.getArgIdx() = getRegionIndex(parent, r);
212 }
213
214 // next parent
215 o = parent;
216 }
217
218 // Determine whether a region-argument component is not needed
219 for (Component &c : components)
220 c.skipIf(c.isRegionArgOfOp && c.isUnique);
221
222 // Find runs of nested loops and determine each loop's depth in the loop nest
223 size_t numSurroundingLoops = 0;
224 for (Component &c : llvm::reverse(components)) {
225 if (c.skip)
226 continue;
227
228 // non-skipped multi-argument operands interrupt the loop nest
229 if (c.isRegionArgOfOp) {
230 numSurroundingLoops = 0;
231 continue;
232 }
233
234 // Multiple loops in a region means each of them is the outermost loop of a
235 // new loop nest
236 if (!c.isUnique)
237 numSurroundingLoops = 0;
238
239 c.getLoopDepth() = numSurroundingLoops;
240
241 // Next loop is surrounded by one more loop
242 if (isa<CanonicalLoopOp>(c.getContainerOp()))
243 numSurroundingLoops += 1;
244 }
245
246 // In loop nests, skip all but the innermost loop that contains the depth
247 // number
248 bool isLoopNest = false;
249 for (Component &c : components) {
250 if (c.skip || c.isRegionArgOfOp)
251 continue;
252
253 if (!isLoopNest && c.getLoopDepth() >= 1) {
254 // Innermost loop of a loop nest of at least two loops
255 isLoopNest = true;
256 } else if (isLoopNest) {
257 // Non-innermost loop of a loop nest
258 c.skipIf(c.isUnique);
259
260 // If there is no surrounding loop left, this must have been the outermost
261 // loop; leave loop-nest mode for the next iteration
262 if (c.getLoopDepth() == 0)
263 isLoopNest = false;
264 }
265 }
266
267 // Skip non-loop unambiguous regions (but they should interrupt loop nests, so
268 // we mark them as skipped only after computing loop nests)
269 for (Component &c : components)
270 c.skipIf(!c.isRegionArgOfOp && c.isUnique &&
271 !isa<CanonicalLoopOp>(c.getContainerOp()));
272
273 // Components can be skipped if they are already disambiguated by their parent
274 // (or does not have a parent)
275 bool newRegion = true;
276 for (Component &c : llvm::reverse(components)) {
277 c.skipIf(newRegion && c.isUnique);
278
279 // non-skipped components disambiguate unique children
280 if (!c.skip)
281 newRegion = true;
282
283 // ...except canonical loops that need a suffix for each nest
284 if (!c.isRegionArgOfOp && c.getContainerOp())
285 newRegion = false;
286 }
287
288 // Compile the nesting name string
289 SmallString<64> Name{prefix};
290 llvm::raw_svector_ostream NameOS(Name);
291 for (auto &c : llvm::reverse(components)) {
292 if (c.skip)
293 continue;
294
295 if (c.isRegionArgOfOp)
296 NameOS << "_r" << c.getArgIdx();
297 else if (c.getLoopDepth() >= 1)
298 NameOS << "_d" << c.getLoopDepth();
299 else
300 NameOS << "_s" << c.getOpPos();
301 }
302
303 return NameOS.str().str();
304}
305
306void OpenMPDialect::initialize() {
307 addOperations<
308#define GET_OP_LIST
309#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
310 >();
311 addAttributes<
312#define GET_ATTRDEF_LIST
313#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
314 >();
315 addTypes<
316#define GET_TYPEDEF_LIST
317#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
318 >();
319
320 declarePromisedInterface<ConvertToLLVMPatternInterface, OpenMPDialect>();
321
322 MemRefType::attachInterface<MemRefPointerLikeModel>(*getContext());
323 LLVM::LLVMPointerType::attachInterface<LLVMPointerPointerLikeModel>(
324 *getContext());
325
326 // Attach default offload module interface to module op to access
327 // offload functionality through
328 mlir::ModuleOp::attachInterface<mlir::omp::OffloadModuleDefaultModel>(
329 *getContext());
330
331 // Attach default declare target interfaces to operations which can be marked
332 // as declare target (Global Operations and Functions/Subroutines in dialects
333 // that Fortran (or other languages that lower to MLIR) translates too
334 mlir::LLVM::GlobalOp::attachInterface<
336 *getContext());
337 mlir::LLVM::LLVMFuncOp::attachInterface<
339 *getContext());
340 mlir::func::FuncOp::attachInterface<
342}
343
344//===----------------------------------------------------------------------===//
345// Dialect operation attribute verification
346//===----------------------------------------------------------------------===//
347
348static LogicalResult verifyDeclareTargetAttr(Operation *op, Attribute attr) {
349 if (!isa<DeclareTargetInterface>(op))
350 return op->emitError() << "omp.declare_target can only be applied to "
351 "DeclareTargetInterface ops";
352
353 auto declareTargetAttr = dyn_cast<DeclareTargetAttr>(attr);
354 if (!declareTargetAttr)
355 return op->emitError()
356 << "omp.declare_target must be an #omp.declaretarget attribute";
357
358 if (isa<mlir::FunctionOpInterface>(op)) {
359 if (declareTargetAttr.getAutomap())
360 return op->emitOpError()
361 << "omp.declare_target 'automap' is not valid on functions";
362
363 // TODO: Disallow the `local` clause (OpenMP 6.0).
364 if (declareTargetAttr.getCaptureClause() ==
365 mlir::omp::DeclareTargetCaptureClause::link)
366 return op->emitOpError()
367 << "omp.declare_target 'link' is not valid on functions";
368 } else {
369 // TODO: Disallow the `indirect` clause (OpenMP 5.1).
370 if (declareTargetAttr.getImplicit())
371 return op->emitOpError()
372 << "omp.declare_target 'implicit' is only valid on functions";
373 }
374 return success();
375}
376
377LogicalResult
378OpenMPDialect::verifyOperationAttribute(Operation *op,
379 NamedAttribute attribute) {
380 if (attribute.getName() == "omp.declare_target")
381 return verifyDeclareTargetAttr(op, attribute.getValue());
382
383 return success();
384}
385
386//===----------------------------------------------------------------------===//
387// Parser and printer for Allocate Clause
388//===----------------------------------------------------------------------===//
389
390/// Parse an allocate clause with allocators and a list of operands with types.
391///
392/// allocate-operand-list :: = allocate-operand |
393/// allocator-operand `,` allocate-operand-list
394/// allocate-operand :: = ssa-id-and-type -> ssa-id-and-type
395/// ssa-id-and-type ::= ssa-id `:` type
396static ParseResult parseAllocateAndAllocator(
397 OpAsmParser &parser,
399 SmallVectorImpl<Type> &allocateTypes,
401 SmallVectorImpl<Type> &allocatorTypes) {
402
403 return parser.parseCommaSeparatedList([&]() {
405 Type type;
406 if (parser.parseOperand(operand) || parser.parseColonType(type))
407 return failure();
408 allocatorVars.push_back(operand);
409 allocatorTypes.push_back(type);
410 if (parser.parseArrow())
411 return failure();
412 if (parser.parseOperand(operand) || parser.parseColonType(type))
413 return failure();
414
415 allocateVars.push_back(operand);
416 allocateTypes.push_back(type);
417 return success();
418 });
419}
420
421/// Print allocate clause
423 OperandRange allocateVars,
424 TypeRange allocateTypes,
425 OperandRange allocatorVars,
426 TypeRange allocatorTypes) {
427 for (unsigned i = 0; i < allocateVars.size(); ++i) {
428 std::string separator = i == allocateVars.size() - 1 ? "" : ", ";
429 p << allocatorVars[i] << " : " << allocatorTypes[i] << " -> ";
430 p << allocateVars[i] << " : " << allocateTypes[i] << separator;
431 }
432}
433
434//===----------------------------------------------------------------------===//
435// Parser and printer for a clause attribute (StringEnumAttr)
436//===----------------------------------------------------------------------===//
437
438template <typename ClauseAttr>
439static ParseResult parseClauseAttr(AsmParser &parser, ClauseAttr &attr) {
440 using ClauseT = decltype(std::declval<ClauseAttr>().getValue());
441 StringRef enumStr;
442 SMLoc loc = parser.getCurrentLocation();
443 if (parser.parseKeyword(&enumStr))
444 return failure();
445 if (std::optional<ClauseT> enumValue = symbolizeEnum<ClauseT>(enumStr)) {
446 attr = ClauseAttr::get(parser.getContext(), *enumValue);
447 return success();
448 }
449 return parser.emitError(loc, "invalid clause value: '") << enumStr << "'";
450}
451
452template <typename ClauseAttr>
453static void printClauseAttr(OpAsmPrinter &p, Operation *op, ClauseAttr attr) {
454 p << stringifyEnum(attr.getValue());
455}
456
457//===----------------------------------------------------------------------===//
458// Parser and printer for Linear Clause
459//===----------------------------------------------------------------------===//
460
461/// linear ::= `linear` `(` linear-list `)`
462/// linear-list := linear-val | linear-val linear-list
463/// linear-val := ssa-id-and-type `=` ssa-id-and-type
464/// | `val` `(` ssa-id-and-type `=` ssa-id-and-type `)`
465/// | `ref` `(` ssa-id-and-type `=` ssa-id-and-type `)`
466/// | `uval` `(` ssa-id-and-type `=` ssa-id-and-type `)`
467static ParseResult parseLinearClause(
468 OpAsmParser &parser,
470 SmallVectorImpl<Type> &linearTypes,
472 SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearModifiers,
473 ArrayAttr &linearSyms) {
474 SmallVector<Attribute> modifiers;
475 auto result = parser.parseCommaSeparatedList([&]() {
477 Type type, stepType;
479
480 std::optional<omp::LinearModifier> linearModifier;
481 if (succeeded(parser.parseOptionalKeyword("val"))) {
482 linearModifier = omp::LinearModifier::val;
483 } else if (succeeded(parser.parseOptionalKeyword("ref"))) {
484 linearModifier = omp::LinearModifier::ref;
485 } else if (succeeded(parser.parseOptionalKeyword("uval"))) {
486 linearModifier = omp::LinearModifier::uval;
487 }
488
489 bool hasLinearModifierParens = linearModifier.has_value();
490 if (hasLinearModifierParens && parser.parseLParen())
491 return failure();
492
493 if (parser.parseOperand(var) || parser.parseColonType(type) ||
494 parser.parseEqual() || parser.parseOperand(stepVar) ||
495 parser.parseColonType(stepType))
496 return failure();
497
498 if (hasLinearModifierParens && parser.parseRParen())
499 return failure();
500
501 linearVars.push_back(var);
502 linearTypes.push_back(type);
503 linearStepVars.push_back(stepVar);
504 linearStepTypes.push_back(stepType);
505 if (linearModifier) {
506 modifiers.push_back(
507 omp::LinearModifierAttr::get(parser.getContext(), *linearModifier));
508 } else {
509 modifiers.push_back(UnitAttr::get(parser.getContext()));
510 }
511 return success();
512 });
513 if (failed(result))
514 return failure();
515 linearModifiers = ArrayAttr::get(parser.getContext(), modifiers);
516 return success();
517}
518
519/// Print Linear Clause. `linearSyms` is only used by ops with a region, which
520/// print the linear clause with their entry block arguments.
522 ValueRange linearVars, TypeRange linearTypes,
523 ValueRange linearStepVars, TypeRange stepVarTypes,
524 ArrayAttr linearModifiers, ArrayAttr linearSyms) {
525 size_t linearVarsSize = linearVars.size();
526 for (unsigned i = 0; i < linearVarsSize; ++i) {
527 if (i != 0)
528 p << ", ";
529 // Print modifier keyword wrapper if present.
530 Attribute modAttr = linearModifiers ? linearModifiers[i] : nullptr;
531 auto mod = modAttr ? dyn_cast<omp::LinearModifierAttr>(modAttr) : nullptr;
532 if (mod) {
533 p << omp::stringifyLinearModifier(mod.getValue()) << "(";
534 }
535 p << linearVars[i] << " : " << linearTypes[i];
536 p << " = " << linearStepVars[i] << " : " << stepVarTypes[i];
537 if (mod)
538 p << ")";
539 }
540}
541
542//===----------------------------------------------------------------------===//
543// Verifier for Linear modifier
544//===----------------------------------------------------------------------===//
545
546/// OpenMP 5.2, Section 5.4.6: "A linear-modifier may be specified as ref or
547/// uval only on a declare simd directive."
548/// Also verifies that modifier count matches variable count.
549static LogicalResult
550verifyLinearModifiers(Operation *op, std::optional<ArrayAttr> linearModifiers,
551 OperandRange linearVars, bool isDeclareSimd = false) {
552 if (!linearModifiers)
553 return success();
554 if (linearModifiers->size() != linearVars.size())
555 return op->emitOpError()
556 << "expected as many linear modifiers as linear variables";
557 if (!isDeclareSimd) {
558 for (Attribute attr : *linearModifiers) {
559 if (!attr)
560 continue;
561 auto modAttr = dyn_cast<omp::LinearModifierAttr>(attr);
562 if (!modAttr)
563 continue;
564 omp::LinearModifier mod = modAttr.getValue();
565 if (mod == omp::LinearModifier::ref || mod == omp::LinearModifier::uval)
566 return op->emitOpError()
567 << "linear modifier '" << omp::stringifyLinearModifier(mod)
568 << "' may only be specified on a declare simd directive";
569 }
570 }
571 return success();
572}
573
574//===----------------------------------------------------------------------===//
575// Verifier for Nontemporal Clause
576//===----------------------------------------------------------------------===//
577
578static LogicalResult verifyNontemporalClause(Operation *op,
579 OperandRange nontemporalVars) {
580
581 // Check if each var is unique - OpenMP 5.0 -> 2.9.3.1 section
582 DenseSet<Value> nontemporalItems;
583 for (const auto &it : nontemporalVars)
584 if (!nontemporalItems.insert(it).second)
585 return op->emitOpError() << "nontemporal variable used more than once";
586
587 return success();
588}
589
590//===----------------------------------------------------------------------===//
591// Parser, verifier and printer for Aligned Clause
592//===----------------------------------------------------------------------===//
593static LogicalResult verifyAlignedClause(Operation *op,
594 std::optional<ArrayAttr> alignments,
595 OperandRange alignedVars) {
596 // Check if number of alignment values equals to number of aligned variables
597 if (!alignedVars.empty()) {
598 if (!alignments || alignments->size() != alignedVars.size())
599 return op->emitOpError()
600 << "expected as many alignment values as aligned variables";
601 } else {
602 if (alignments)
603 return op->emitOpError() << "unexpected alignment values attribute";
604 return success();
605 }
606
607 // Check if each var is aligned only once - OpenMP 4.5 -> 2.8.1 section
608 DenseSet<Value> alignedItems;
609 for (auto it : alignedVars)
610 if (!alignedItems.insert(it).second)
611 return op->emitOpError() << "aligned variable used more than once";
612
613 if (!alignments)
614 return success();
615
616 // Check if all alignment values are positive - OpenMP 4.5 -> 2.8.1 section
617 for (unsigned i = 0; i < (*alignments).size(); ++i) {
618 if (auto intAttr = llvm::dyn_cast<IntegerAttr>((*alignments)[i])) {
619 if (intAttr.getValue().sle(0))
620 return op->emitOpError() << "alignment should be greater than 0";
621 } else {
622 return op->emitOpError() << "expected integer alignment";
623 }
624 }
625
626 return success();
627}
628
629/// aligned ::= `aligned` `(` aligned-list `)`
630/// aligned-list := aligned-val | aligned-val aligned-list
631/// aligned-val := ssa-id-and-type `->` alignment
632static ParseResult
635 SmallVectorImpl<Type> &alignedTypes,
636 ArrayAttr &alignmentsAttr) {
637 SmallVector<Attribute> alignmentVec;
638 if (failed(parser.parseCommaSeparatedList([&]() {
639 if (parser.parseOperand(alignedVars.emplace_back()) ||
640 parser.parseColonType(alignedTypes.emplace_back()) ||
641 parser.parseArrow() ||
642 parser.parseAttribute(alignmentVec.emplace_back())) {
643 return failure();
644 }
645 return success();
646 })))
647 return failure();
648 SmallVector<Attribute> alignments(alignmentVec.begin(), alignmentVec.end());
649 alignmentsAttr = ArrayAttr::get(parser.getContext(), alignments);
650 return success();
651}
652
653/// Print Aligned Clause
655 ValueRange alignedVars, TypeRange alignedTypes,
656 std::optional<ArrayAttr> alignments) {
657 for (unsigned i = 0; i < alignedVars.size(); ++i) {
658 if (i != 0)
659 p << ", ";
660 p << alignedVars[i] << " : " << alignedVars[i].getType();
661 p << " -> " << (*alignments)[i];
662 }
663}
664
665static LogicalResult verifyAllocateClause(
666 Operation *op, ValueRange allocateVars, ValueRange allocatorVars,
667 DenseI64ArrayAttr allocateAlignments,
668 DenseI64ArrayAttr allocatePrivateIndices, ValueRange privateVars = {},
669 ArrayAttr privateSyms = nullptr, bool requirePrivateIndices = false) {
670 if (allocateVars.size() != allocatorVars.size())
671 return op->emitError(
672 "expected equal sizes for allocate and allocator variables");
673
674 if (allocateVars.empty()) {
675 if (allocateAlignments)
676 return op->emitError(
677 "unexpected allocate alignments without allocate variables");
678 if (allocatePrivateIndices)
679 return op->emitError(
680 "unexpected allocate private indices without allocate variables");
681 return success();
682 }
683
684 if (allocateAlignments) {
685 ArrayRef<int64_t> alignments = allocateAlignments.asArrayRef();
686 if (alignments.size() != allocateVars.size())
687 return op->emitError(
688 "expected as many allocate alignments as allocate variables");
689 for (int64_t alignment : alignments) {
690 if (alignment < 0)
691 return op->emitError("expected non-negative allocate alignments");
692 if (alignment != 0 && (alignment & (alignment - 1)) != 0)
693 return op->emitError(
694 "expected positive allocate alignments to be powers of two");
695 }
696 }
697
698 if (!allocatePrivateIndices) {
699 if (requirePrivateIndices)
700 return op->emitError(
701 "expected an allocate private index for each allocate variable");
702 return success();
703 }
704
705 ArrayRef<int64_t> indices = allocatePrivateIndices.asArrayRef();
706 if (indices.size() != allocateVars.size())
707 return op->emitError(
708 "expected as many allocate private indices as allocate variables");
709
710 DenseSet<int64_t> usedPrivateSlots;
711 for (auto [allocateVar, privateIndex] :
712 llvm::zip_equal(allocateVars, indices)) {
713 if (privateIndex < 0 ||
714 static_cast<uint64_t>(privateIndex) >= privateVars.size())
715 return op->emitError("allocate private index is out of range");
716 if (!usedPrivateSlots.insert(privateIndex).second)
717 return op->emitError(
718 "allocate private index refers to a private variable more than once");
719
720 Value privateVar = privateVars[privateIndex];
721 if (allocateVar.getType() != privateVar.getType())
722 return op->emitError()
723 << "type mismatch between allocate variable and private variable "
724 "at index "
725 << privateIndex;
726 if (allocateVar != privateVar)
727 return op->emitError()
728 << "allocate variable does not match private variable at index "
729 << privateIndex;
730
731 if (!privateSyms ||
732 static_cast<uint64_t>(privateIndex) >= privateSyms.size())
733 return op->emitError(
734 "allocate private index does not have a privatizer symbol");
735
736 auto privateSym = dyn_cast<SymbolRefAttr>(privateSyms[privateIndex]);
737 if (!privateSym)
738 return op->emitError(
739 "allocate private index does not reference a privatizer symbol");
740 PrivateClauseOp privatizer =
742 if (!privatizer)
743 return op->emitError() << "failed to lookup privatizer op with symbol: '"
744 << privateSym << "'";
745 if (privatizer.getDataSharingType() != DataSharingClauseType::Private &&
746 privatizer.getDataSharingType() != DataSharingClauseType::FirstPrivate)
747 return op->emitError(
748 "allocate private index must refer to private or firstprivate "
749 "storage");
750 }
751
752 return success();
753}
754
755//===----------------------------------------------------------------------===//
756// Parser, printer and verifier for Schedule Clause
757//===----------------------------------------------------------------------===//
758
759static ParseResult
761 SmallVectorImpl<SmallString<12>> &modifiers) {
762 if (modifiers.size() > 2)
763 return parser.emitError(parser.getNameLoc()) << " unexpected modifier(s)";
764 for (const auto &mod : modifiers) {
765 // Translate the string. If it has no value, then it was not a valid
766 // modifier!
767 auto symbol = symbolizeScheduleModifier(mod);
768 if (!symbol)
769 return parser.emitError(parser.getNameLoc())
770 << " unknown modifier type: " << mod;
771 }
772
773 // If we have one modifier that is "simd", then stick a "none" modiifer in
774 // index 0.
775 if (modifiers.size() == 1) {
776 if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd) {
777 modifiers.push_back(modifiers[0]);
778 modifiers[0] = stringifyScheduleModifier(ScheduleModifier::none);
779 }
780 } else if (modifiers.size() == 2) {
781 // If there are two modifier:
782 // First modifier should not be simd, second one should be simd
783 if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd ||
784 symbolizeScheduleModifier(modifiers[1]) != ScheduleModifier::simd)
785 return parser.emitError(parser.getNameLoc())
786 << " incorrect modifier order";
787 }
788 return success();
789}
790
791/// schedule ::= `schedule` `(` sched-list `)`
792/// sched-list ::= sched-val | sched-val sched-list |
793/// sched-val `,` sched-modifier
794/// sched-val ::= sched-with-chunk | sched-wo-chunk
795/// sched-with-chunk ::= sched-with-chunk-types (`=` ssa-id-and-type)?
796/// sched-with-chunk-types ::= `static` | `dynamic` | `guided`
797/// sched-wo-chunk ::= `auto` | `runtime`
798/// sched-modifier ::= sched-mod-val | sched-mod-val `,` sched-mod-val
799/// sched-mod-val ::= `monotonic` | `nonmonotonic` | `simd` | `none`
800static ParseResult
801parseScheduleClause(OpAsmParser &parser, ClauseScheduleKindAttr &scheduleAttr,
802 ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd,
803 std::optional<OpAsmParser::UnresolvedOperand> &chunkSize,
804 Type &chunkType) {
805 StringRef keyword;
806 if (parser.parseKeyword(&keyword))
807 return failure();
808 std::optional<mlir::omp::ClauseScheduleKind> schedule =
809 symbolizeClauseScheduleKind(keyword);
810 if (!schedule)
811 return parser.emitError(parser.getNameLoc()) << " expected schedule kind";
812
813 scheduleAttr = ClauseScheduleKindAttr::get(parser.getContext(), *schedule);
814 switch (*schedule) {
815 case ClauseScheduleKind::Static:
816 case ClauseScheduleKind::Dynamic:
817 case ClauseScheduleKind::Guided:
818 if (succeeded(parser.parseOptionalEqual())) {
819 chunkSize = OpAsmParser::UnresolvedOperand{};
820 if (parser.parseOperand(*chunkSize) || parser.parseColonType(chunkType))
821 return failure();
822 } else {
823 chunkSize = std::nullopt;
824 }
825 break;
826 case ClauseScheduleKind::Auto:
827 case ClauseScheduleKind::Runtime:
828 case ClauseScheduleKind::Distribute:
829 chunkSize = std::nullopt;
830 }
831
832 // If there is a comma, we have one or more modifiers..
834 while (succeeded(parser.parseOptionalComma())) {
835 StringRef mod;
836 if (parser.parseKeyword(&mod))
837 return failure();
838 modifiers.push_back(mod);
839 }
840
841 if (verifyScheduleModifiers(parser, modifiers))
842 return failure();
843
844 if (!modifiers.empty()) {
845 SMLoc loc = parser.getCurrentLocation();
846 if (std::optional<ScheduleModifier> mod =
847 symbolizeScheduleModifier(modifiers[0])) {
848 scheduleMod = ScheduleModifierAttr::get(parser.getContext(), *mod);
849 } else {
850 return parser.emitError(loc, "invalid schedule modifier");
851 }
852 // Only SIMD attribute is allowed here!
853 if (modifiers.size() > 1) {
854 assert(symbolizeScheduleModifier(modifiers[1]) == ScheduleModifier::simd);
855 scheduleSimd = UnitAttr::get(parser.getBuilder().getContext());
856 }
857 }
858
859 return success();
860}
861
862/// Print schedule clause
864 ClauseScheduleKindAttr scheduleKind,
865 ScheduleModifierAttr scheduleMod,
866 UnitAttr scheduleSimd, Value scheduleChunk,
867 Type scheduleChunkType) {
868 p << stringifyClauseScheduleKind(scheduleKind.getValue());
869 if (scheduleChunk)
870 p << " = " << scheduleChunk << " : " << scheduleChunk.getType();
871 if (scheduleMod)
872 p << ", " << stringifyScheduleModifier(scheduleMod.getValue());
873 if (scheduleSimd)
874 p << ", simd";
875}
876
877//===----------------------------------------------------------------------===//
878// Parser and printer for Order Clause
879//===----------------------------------------------------------------------===//
880
881// order ::= `order` `(` [order-modifier ':'] concurrent `)`
882// order-modifier ::= reproducible | unconstrained
883static ParseResult parseOrderClause(OpAsmParser &parser,
884 ClauseOrderKindAttr &order,
885 OrderModifierAttr &orderMod) {
886 StringRef enumStr;
887 SMLoc loc = parser.getCurrentLocation();
888 if (parser.parseKeyword(&enumStr))
889 return failure();
890 if (std::optional<OrderModifier> enumValue =
891 symbolizeOrderModifier(enumStr)) {
892 orderMod = OrderModifierAttr::get(parser.getContext(), *enumValue);
893 if (parser.parseOptionalColon())
894 return failure();
895 loc = parser.getCurrentLocation();
896 if (parser.parseKeyword(&enumStr))
897 return failure();
898 }
899 if (std::optional<ClauseOrderKind> enumValue =
900 symbolizeClauseOrderKind(enumStr)) {
901 order = ClauseOrderKindAttr::get(parser.getContext(), *enumValue);
902 return success();
903 }
904 return parser.emitError(loc, "invalid clause value: '") << enumStr << "'";
905}
906
908 ClauseOrderKindAttr order,
909 OrderModifierAttr orderMod) {
910 if (orderMod)
911 p << stringifyOrderModifier(orderMod.getValue()) << ":";
912 if (order)
913 p << stringifyClauseOrderKind(order.getValue());
914}
915
916template <typename ClauseTypeAttr, typename ClauseType>
917static ParseResult
918parseGranularityClause(OpAsmParser &parser, ClauseTypeAttr &prescriptiveness,
919 std::optional<OpAsmParser::UnresolvedOperand> &operand,
920 Type &operandType,
921 std::optional<ClauseType> (*symbolizeClause)(StringRef),
922 StringRef clauseName) {
923 StringRef enumStr;
924 if (succeeded(parser.parseOptionalKeyword(&enumStr))) {
925 if (std::optional<ClauseType> enumValue = symbolizeClause(enumStr)) {
926 prescriptiveness = ClauseTypeAttr::get(parser.getContext(), *enumValue);
927 if (parser.parseComma())
928 return failure();
929 } else {
930 return parser.emitError(parser.getCurrentLocation())
931 << "invalid " << clauseName << " modifier : '" << enumStr << "'";
932 ;
933 }
934 }
935
937 if (succeeded(parser.parseOperand(var))) {
938 operand = var;
939 } else {
940 return parser.emitError(parser.getCurrentLocation())
941 << "expected " << clauseName << " operand";
942 }
943
944 if (operand.has_value()) {
945 if (parser.parseColonType(operandType))
946 return failure();
947 }
948
949 return success();
950}
951
952template <typename ClauseTypeAttr, typename ClauseType>
953static void
955 ClauseTypeAttr prescriptiveness, Value operand,
956 mlir::Type operandType,
957 StringRef (*stringifyClauseType)(ClauseType)) {
958
959 if (prescriptiveness)
960 p << stringifyClauseType(prescriptiveness.getValue()) << ", ";
961
962 if (operand)
963 p << operand << ": " << operandType;
964}
965
966//===----------------------------------------------------------------------===//
967// Parser and printer for grainsize Clause
968//===----------------------------------------------------------------------===//
969
970// grainsize ::= `grainsize` `(` [strict ':'] grain-size `)`
971static ParseResult
972parseGrainsizeClause(OpAsmParser &parser, ClauseGrainsizeTypeAttr &grainsizeMod,
973 std::optional<OpAsmParser::UnresolvedOperand> &grainsize,
974 Type &grainsizeType) {
976 parser, grainsizeMod, grainsize, grainsizeType,
977 &symbolizeClauseGrainsizeType, "grainsize");
978}
979
981 ClauseGrainsizeTypeAttr grainsizeMod,
982 Value grainsize, mlir::Type grainsizeType) {
984 p, op, grainsizeMod, grainsize, grainsizeType,
985 &stringifyClauseGrainsizeType);
986}
987
988//===----------------------------------------------------------------------===//
989// Parser and printer for num_tasks Clause
990//===----------------------------------------------------------------------===//
991
992// numtask ::= `num_tasks` `(` [strict ':'] num-tasks `)`
993static ParseResult
994parseNumTasksClause(OpAsmParser &parser, ClauseNumTasksTypeAttr &numTasksMod,
995 std::optional<OpAsmParser::UnresolvedOperand> &numTasks,
996 Type &numTasksType) {
998 parser, numTasksMod, numTasks, numTasksType, &symbolizeClauseNumTasksType,
999 "num_tasks");
1000}
1001
1003 ClauseNumTasksTypeAttr numTasksMod,
1004 Value numTasks, mlir::Type numTasksType) {
1006 p, op, numTasksMod, numTasks, numTasksType, &stringifyClauseNumTasksType);
1007}
1008
1009//===----------------------------------------------------------------------===//
1010// Parser and printer for Heap Alloc Clause
1011//===----------------------------------------------------------------------===//
1012
1013/// operation ::= $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
1014static ParseResult parseHeapAllocClause(
1015 OpAsmParser &parser, TypeAttr &inTypeAttr,
1017 SmallVectorImpl<Type> &typeparamsTypes,
1019 SmallVectorImpl<Type> &shapeTypes) {
1020 mlir::Type inType;
1021 if (parser.parseType(inType))
1022 return mlir::failure();
1023 inTypeAttr = TypeAttr::get(inType);
1024
1025 if (!parser.parseOptionalLParen()) {
1026 // parse the LEN params of the derived type. (<params> : <types>)
1027 if (parser.parseOperandList(typeparams, OpAsmParser::Delimiter::None) ||
1028 parser.parseColonTypeList(typeparamsTypes) || parser.parseRParen())
1029 return failure();
1030 }
1031
1032 if (!parser.parseOptionalComma()) {
1033 // parse size to scale by, vector of n dimensions of type index
1035 return failure();
1036
1037 // TODO: This overrides the actual types of the operands, which might cause
1038 // issues when they don't match. At the moment this is done in place of
1039 // making the corresponding operand type `Variadic<Index>` because index
1040 // types are lowered to I64 prior to LLVM IR translation.
1041 shapeTypes.append(shape.size(), IndexType::get(parser.getContext()));
1042 }
1043
1044 return success();
1045}
1046
1048 TypeAttr inType, ValueRange typeparams,
1049 TypeRange typeparamsTypes, ValueRange shape,
1050 TypeRange shapeTypes) {
1051 p << inType;
1052 if (!typeparams.empty()) {
1053 p << '(' << typeparams << " : " << typeparamsTypes << ')';
1054 }
1055 for (auto sh : shape) {
1056 p << ", ";
1057 p.printOperand(sh);
1058 }
1059}
1060
1061//===----------------------------------------------------------------------===//
1062// Parser, printer and verify for dyn_groupprivate Clause
1063//===----------------------------------------------------------------------===//
1064
1065static LogicalResult
1066verifyDynGroupprivateClause(Operation *op, AccessGroupModifierAttr accessGroup,
1067 FallbackModifierAttr fallback,
1068 Value dynGroupprivateSize) {
1069 if (!dynGroupprivateSize && (accessGroup || fallback))
1070 return op->emitOpError("dyn_groupprivate modifiers require a size operand");
1071
1072 return success();
1073}
1074
1076 OpAsmParser &parser, AccessGroupModifierAttr &accessGroupAttr,
1077 FallbackModifierAttr &fallbackAttr,
1078 std::optional<OpAsmParser::UnresolvedOperand> &dynGroupprivateSize,
1079 Type &sizeType) {
1080
1081 bool parsedAccessGroup = false;
1082 bool parsedFallback = false;
1083 bool parsedSize = false;
1084
1085 return parser.parseCommaSeparatedList([&]() -> ParseResult {
1086 // Parse AccessGroupModifier.
1087 if (succeeded(parser.parseOptionalKeyword("cgroup"))) {
1088 if (parsedAccessGroup)
1089 return parser.emitError(parser.getCurrentLocation(),
1090 "duplicate access group modifier");
1091 accessGroupAttr = AccessGroupModifierAttr::get(
1092 parser.getContext(), AccessGroupModifier::cgroup);
1093 parsedAccessGroup = true;
1094 return success();
1095 }
1096 // Parse FallbackModifier.
1097 if (succeeded(parser.parseOptionalKeyword("fallback"))) {
1098 if (parsedFallback)
1099 return parser.emitError(parser.getCurrentLocation(),
1100 "duplicate fallback modifier");
1101 if (parser.parseLParen())
1102 return parser.emitError(parser.getCurrentLocation(),
1103 "expected '(' after 'fallback'");
1104 llvm::StringRef fbKind;
1105 if (parser.parseKeyword(&fbKind))
1106 return parser.emitError(
1107 parser.getCurrentLocation(),
1108 "expected fallback modifier (abort/null/default_mem)");
1109 std::optional<FallbackModifier> fbEnum;
1110 if (fbKind == "abort")
1111 fbEnum = FallbackModifier::abort;
1112 else if (fbKind == "null")
1113 fbEnum = FallbackModifier::null;
1114 else if (fbKind == "default_mem")
1115 fbEnum = FallbackModifier::default_mem;
1116 else
1117 return parser.emitError(parser.getCurrentLocation(),
1118 "invalid fallback modifier '" + fbKind + "'");
1119 fallbackAttr = FallbackModifierAttr::get(parser.getContext(), *fbEnum);
1120 if (parser.parseRParen())
1121 return parser.emitError(parser.getCurrentLocation(),
1122 "expected ')' after fallback modifier");
1123 parsedFallback = true;
1124 return success();
1125 }
1126 // Parse size operand.
1128 if (succeeded(parser.parseOperand(operand))) {
1129 if (parsedSize)
1130 return parser.emitError(parser.getCurrentLocation(),
1131 "duplicate size operand");
1132 dynGroupprivateSize = operand;
1133 parsedSize = true;
1134 if (failed(parser.parseColon()) || failed(parser.parseType(sizeType)))
1135 return parser.emitError(parser.getCurrentLocation(),
1136 "expected ':' and type after size operand");
1137 return success();
1138 }
1139 return parser.emitError(parser.getCurrentLocation(),
1140 "expected dyn_groupprivate_size operand");
1141 });
1142}
1143
1145 AccessGroupModifierAttr modifierFirst,
1146 FallbackModifierAttr modifierSecond,
1147 Value dynGroupprivateSize,
1148 Type sizeType) {
1149
1150 bool needsComma = false;
1151
1152 if (modifierFirst) {
1153 printer << modifierFirst.getValue();
1154 needsComma = true;
1155 }
1156
1157 if (modifierSecond) {
1158 if (needsComma)
1159 printer << ", ";
1160 printer << "fallback(";
1161 printer << modifierSecond.getValue();
1162 printer << ")";
1163 needsComma = true;
1164 }
1165
1166 if (dynGroupprivateSize) {
1167 if (needsComma)
1168 printer << ", ";
1169 printer << dynGroupprivateSize << " : " << sizeType;
1170 }
1171}
1172
1173//===----------------------------------------------------------------------===//
1174// Parser and printer for in_reduction Clause
1175//===----------------------------------------------------------------------===//
1176
1177/// Parses an `in_reduction` clause for an operation that does not give its
1178/// list items entry block arguments (e.g. `omp.target`). The expected format is
1179/// a comma-separated list of `[byref] @sym %var` followed by `: types`.
1180static ParseResult parseInReductionClause(
1181 OpAsmParser &parser,
1183 SmallVectorImpl<Type> &inReductionTypes,
1184 DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms) {
1186 SmallVector<bool> isByRefVec;
1187
1188 if (parser.parseCommaSeparatedList([&]() {
1189 isByRefVec.push_back(parser.parseOptionalKeyword("byref").succeeded());
1190 if (parser.parseAttribute(symbolVec.emplace_back()) ||
1191 parser.parseOperand(inReductionVars.emplace_back()))
1192 return failure();
1193 return success();
1194 }))
1195 return failure();
1196
1197 if (parser.parseColon())
1198 return failure();
1199
1200 if (parser.parseCommaSeparatedList(
1201 [&]() { return parser.parseType(inReductionTypes.emplace_back()); }))
1202 return failure();
1203
1204 if (inReductionVars.size() != inReductionTypes.size())
1205 return failure();
1206
1207 inReductionByref = makeDenseBoolArrayAttr(parser.getContext(), isByRefVec);
1208 SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());
1209 inReductionSyms = ArrayAttr::get(parser.getContext(), symbolAttrs);
1210 return success();
1211}
1212
1213/// Prints an `in_reduction` clause for an operation that does not give its list
1214/// items entry block arguments (e.g. `omp.target`). Mirrors
1215/// `parseInReductionClause`.
1217 ValueRange inReductionVars,
1218 TypeRange inReductionTypes,
1219 DenseBoolArrayAttr inReductionByref,
1220 ArrayAttr inReductionSyms) {
1221 MLIRContext *ctx = op->getContext();
1222
1223 ArrayAttr syms = inReductionSyms;
1224 if (!syms) {
1225 SmallVector<Attribute> values(inReductionVars.size(), nullptr);
1226 syms = ArrayAttr::get(ctx, values);
1227 }
1228
1229 DenseBoolArrayAttr byref = inReductionByref;
1230 if (!byref) {
1231 SmallVector<bool> values(inReductionVars.size(), false);
1232 byref = DenseBoolArrayAttr::get(ctx, values);
1233 }
1234
1235 llvm::interleaveComma(
1236 llvm::zip_equal(inReductionVars, syms.getValue(), byref.asArrayRef()), p,
1237 [&p](auto t) {
1238 auto [var, sym, isByRef] = t;
1239 if (isByRef)
1240 p << "byref ";
1241 if (sym)
1242 p << sym << " ";
1243 p << var;
1244 });
1245 p << " : ";
1246 llvm::interleaveComma(inReductionTypes, p);
1247}
1248
1249//===----------------------------------------------------------------------===//
1250// Parsers for operations including clauses that define entry block arguments.
1251//===----------------------------------------------------------------------===//
1252
1253namespace {
1254struct MapParseArgs {
1255 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1256 SmallVectorImpl<Type> &types;
1257 MapParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1258 SmallVectorImpl<Type> &types)
1259 : vars(vars), types(types) {}
1260};
1261struct PrivateParseArgs {
1262 llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1263 llvm::SmallVectorImpl<Type> &types;
1264 ArrayAttr &syms;
1265 UnitAttr &needsBarrier;
1266 DenseI64ArrayAttr *mapIndices;
1267 PrivateParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1268 SmallVectorImpl<Type> &types, ArrayAttr &syms,
1269 UnitAttr &needsBarrier,
1270 DenseI64ArrayAttr *mapIndices = nullptr)
1271 : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
1272 mapIndices(mapIndices) {}
1273};
1274
1275struct LinearParseArgs {
1276 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1277 SmallVectorImpl<Type> &types;
1278 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars;
1279 SmallVectorImpl<Type> &stepTypes;
1280 ArrayAttr &varTypes;
1281 ArrayAttr &modifiers;
1282 ArrayAttr &syms;
1283 LinearParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1284 SmallVectorImpl<Type> &types,
1285 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars,
1286 SmallVectorImpl<Type> &stepTypes, ArrayAttr &varTypes,
1287 ArrayAttr &modifiers, ArrayAttr &syms)
1288 : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
1289 varTypes(varTypes), modifiers(modifiers), syms(syms) {}
1290};
1291
1292struct ReductionParseArgs {
1293 SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
1294 SmallVectorImpl<Type> &types;
1295 DenseBoolArrayAttr &byref;
1296 ArrayAttr &syms;
1297 ArrayAttr *modifier;
1298 ReductionParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
1299 SmallVectorImpl<Type> &types, DenseBoolArrayAttr &byref,
1300 ArrayAttr &syms, ArrayAttr *mod = nullptr)
1301 : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
1302};
1303
1304struct AllRegionParseArgs {
1305 std::optional<MapParseArgs> hasDeviceAddrArgs;
1306 std::optional<MapParseArgs> hostEvalArgs;
1307 std::optional<ReductionParseArgs> inReductionArgs;
1308 std::optional<LinearParseArgs> linearArgs;
1309 std::optional<MapParseArgs> mapArgs;
1310 std::optional<PrivateParseArgs> privateArgs;
1311 std::optional<ReductionParseArgs> reductionArgs;
1312 std::optional<ReductionParseArgs> taskReductionArgs;
1313 std::optional<MapParseArgs> useDeviceAddrArgs;
1314 std::optional<MapParseArgs> useDevicePtrArgs;
1315};
1316} // namespace
1317
1318static inline constexpr StringRef getPrivateNeedsBarrierSpelling() {
1319 return "private_barrier";
1320}
1321
1322static ParseResult parseClauseWithRegionArgs(
1323 OpAsmParser &parser,
1325 SmallVectorImpl<Type> &types,
1326 SmallVectorImpl<OpAsmParser::Argument> &regionPrivateArgs,
1327 ArrayAttr *symbols = nullptr, DenseI64ArrayAttr *mapIndices = nullptr,
1328 DenseBoolArrayAttr *byref = nullptr, ArrayAttr *modifier = nullptr,
1329 UnitAttr *needsBarrier = nullptr) {
1331 SmallVector<int64_t> mapIndicesVec;
1332 SmallVector<bool> isByRefVec;
1333 unsigned regionArgOffset = regionPrivateArgs.size();
1334
1335 if (parser.parseLParen())
1336 return failure();
1337
1338 SmallVector<Attribute> modifiers;
1339 if (parser.parseCommaSeparatedList([&]() -> ParseResult {
1340 if (modifier) {
1341 ReductionModifier mod = ReductionModifier::defaultmod;
1342 if (succeeded(parser.parseOptionalKeyword("task")))
1343 mod = ReductionModifier::task;
1344 else if (succeeded(parser.parseOptionalKeyword("inscan")))
1345 mod = ReductionModifier::inscan;
1346 else if (succeeded(parser.parseOptionalKeyword("defaultmod")))
1347 mod = ReductionModifier::defaultmod;
1348 modifiers.push_back(
1349 ReductionModifierAttr::get(parser.getContext(), mod));
1350 }
1351 if (byref)
1352 isByRefVec.push_back(
1353 parser.parseOptionalKeyword("byref").succeeded());
1354
1355 if (symbols && parser.parseAttribute(symbolVec.emplace_back()))
1356 return failure();
1357
1358 if (parser.parseOperand(operands.emplace_back()) ||
1359 parser.parseArrow() ||
1360 parser.parseArgument(regionPrivateArgs.emplace_back()))
1361 return failure();
1362
1363 if (mapIndices) {
1364 if (parser.parseOptionalLSquare().succeeded()) {
1365 if (parser.parseKeyword("map_idx") || parser.parseEqual() ||
1366 parser.parseInteger(mapIndicesVec.emplace_back()) ||
1367 parser.parseRSquare())
1368 return failure();
1369 } else {
1370 mapIndicesVec.push_back(-1);
1371 }
1372 }
1373
1374 return success();
1375 }))
1376 return failure();
1377
1378 if (parser.parseColon() || parser.parseCommaSeparatedList([&]() {
1379 return parser.parseType(types.emplace_back());
1380 }))
1381 return failure();
1382
1383 if (llvm::any_of(modifiers, [](Attribute attr) {
1384 return cast<ReductionModifierAttr>(attr).getValue() !=
1385 ReductionModifier::defaultmod;
1386 }))
1387 *modifier = ArrayAttr::get(parser.getContext(), modifiers);
1388
1389 if (operands.size() != types.size())
1390 return failure();
1391
1392 if (parser.parseRParen())
1393 return failure();
1394
1395 if (needsBarrier) {
1397 .succeeded())
1398 *needsBarrier = mlir::UnitAttr::get(parser.getContext());
1399 }
1400
1401 auto *argsBegin = regionPrivateArgs.begin();
1402 MutableArrayRef argsSubrange(argsBegin + regionArgOffset,
1403 argsBegin + regionArgOffset + types.size());
1404 for (auto [prv, type] : llvm::zip_equal(argsSubrange, types)) {
1405 prv.type = type;
1406 }
1407
1408 if (symbols) {
1409 SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());
1410 *symbols = ArrayAttr::get(parser.getContext(), symbolAttrs);
1411 }
1412
1413 if (!mapIndicesVec.empty())
1414 *mapIndices =
1415 mlir::DenseI64ArrayAttr::get(parser.getContext(), mapIndicesVec);
1416
1417 if (byref)
1418 *byref = makeDenseBoolArrayAttr(parser.getContext(), isByRefVec);
1419
1420 return success();
1421}
1422
1423static ParseResult parseBlockArgClause(
1424 OpAsmParser &parser,
1426 StringRef keyword, std::optional<MapParseArgs> mapArgs) {
1427 if (succeeded(parser.parseOptionalKeyword(keyword))) {
1428 if (!mapArgs)
1429 return failure();
1430
1431 if (failed(parseClauseWithRegionArgs(parser, mapArgs->vars, mapArgs->types,
1432 entryBlockArgs)))
1433 return failure();
1434 }
1435 return success();
1436}
1437
1438static ParseResult parseBlockArgClause(
1439 OpAsmParser &parser,
1441 StringRef keyword, std::optional<PrivateParseArgs> privateArgs) {
1442 if (succeeded(parser.parseOptionalKeyword(keyword))) {
1443 if (!privateArgs)
1444 return failure();
1445
1446 if (failed(parseClauseWithRegionArgs(
1447 parser, privateArgs->vars, privateArgs->types, entryBlockArgs,
1448 &privateArgs->syms, privateArgs->mapIndices, /*byref=*/nullptr,
1449 /*modifier=*/nullptr, &privateArgs->needsBarrier)))
1450 return failure();
1451 }
1452 return success();
1453}
1454
1455static ParseResult parseBlockArgClause(
1456 OpAsmParser &parser,
1458 StringRef keyword, std::optional<ReductionParseArgs> reductionArgs) {
1459 if (succeeded(parser.parseOptionalKeyword(keyword))) {
1460 if (!reductionArgs)
1461 return failure();
1462 if (failed(parseClauseWithRegionArgs(
1463 parser, reductionArgs->vars, reductionArgs->types, entryBlockArgs,
1464 &reductionArgs->syms, /*mapIndices=*/nullptr, &reductionArgs->byref,
1465 reductionArgs->modifier)))
1466 return failure();
1467 }
1468 return success();
1469}
1470
1471/// linear-entry := [modifier `(`] symbol ssa-id `->` ssa-id `:` type `=`
1472/// ssa-id `:` type [`)`]
1473static ParseResult parseBlockArgClause(
1474 OpAsmParser &parser,
1476 StringRef keyword, std::optional<LinearParseArgs> linearArgs) {
1477 if (!linearArgs || failed(parser.parseOptionalKeyword(keyword)))
1478 return success();
1479
1480 MLIRContext *ctx = parser.getContext();
1481 SmallVector<Attribute> syms, modifiers;
1482 if (parser.parseLParen() ||
1483 parser.parseCommaSeparatedList([&]() -> ParseResult {
1484 std::optional<LinearModifier> modifier;
1485 if (succeeded(parser.parseOptionalKeyword("val")))
1486 modifier = LinearModifier::val;
1487 else if (succeeded(parser.parseOptionalKeyword("ref")))
1488 modifier = LinearModifier::ref;
1489 else if (succeeded(parser.parseOptionalKeyword("uval")))
1490 modifier = LinearModifier::uval;
1491 if (modifier && parser.parseLParen())
1492 return failure();
1493
1494 SymbolRefAttr sym;
1495 OpAsmParser::Argument &arg = entryBlockArgs.emplace_back();
1496 if (parser.parseAttribute(sym) ||
1497 parser.parseOperand(linearArgs->vars.emplace_back()) ||
1498 parser.parseArrow() || parser.parseArgument(arg) ||
1499 parser.parseColonType(linearArgs->types.emplace_back()) ||
1500 parser.parseEqual() ||
1501 parser.parseOperand(linearArgs->stepVars.emplace_back()) ||
1502 parser.parseColonType(linearArgs->stepTypes.emplace_back()))
1503 return failure();
1504 arg.type = linearArgs->types.back();
1505
1506 if (modifier && parser.parseRParen())
1507 return failure();
1508
1509 syms.push_back(sym);
1510 modifiers.push_back(
1511 modifier ? Attribute(LinearModifierAttr::get(ctx, *modifier))
1512 : Attribute(UnitAttr::get(ctx)));
1513 return success();
1514 }) ||
1515 parser.parseRParen())
1516 return failure();
1517
1518 linearArgs->syms = ArrayAttr::get(ctx, syms);
1519 linearArgs->modifiers = ArrayAttr::get(ctx, modifiers);
1520
1521 if (succeeded(parser.parseOptionalKeyword("linear_var_types")) &&
1522 (parser.parseLParen() || parser.parseAttribute(linearArgs->varTypes) ||
1523 parser.parseRParen()))
1524 return failure();
1525 return success();
1526}
1527
1528static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region &region,
1529 AllRegionParseArgs args) {
1531
1532 if (failed(parseBlockArgClause(parser, entryBlockArgs, "has_device_addr",
1533 args.hasDeviceAddrArgs)))
1534 return parser.emitError(parser.getCurrentLocation())
1535 << "invalid `has_device_addr` format";
1536
1537 if (failed(parseBlockArgClause(parser, entryBlockArgs, "host_eval",
1538 args.hostEvalArgs)))
1539 return parser.emitError(parser.getCurrentLocation())
1540 << "invalid `host_eval` format";
1541
1542 if (failed(parseBlockArgClause(parser, entryBlockArgs, "in_reduction",
1543 args.inReductionArgs)))
1544 return parser.emitError(parser.getCurrentLocation())
1545 << "invalid `in_reduction` format";
1546
1547 if (failed(parseBlockArgClause(parser, entryBlockArgs, "linear",
1548 args.linearArgs)))
1549 return parser.emitError(parser.getCurrentLocation())
1550 << "invalid `linear` format";
1551
1552 if (failed(parseBlockArgClause(parser, entryBlockArgs, "map_entries",
1553 args.mapArgs)))
1554 return parser.emitError(parser.getCurrentLocation())
1555 << "invalid `map_entries` format";
1556
1557 if (failed(parseBlockArgClause(parser, entryBlockArgs, "private",
1558 args.privateArgs)))
1559 return parser.emitError(parser.getCurrentLocation())
1560 << "invalid `private` format";
1561
1562 if (failed(parseBlockArgClause(parser, entryBlockArgs, "reduction",
1563 args.reductionArgs)))
1564 return parser.emitError(parser.getCurrentLocation())
1565 << "invalid `reduction` format";
1566
1567 if (failed(parseBlockArgClause(parser, entryBlockArgs, "task_reduction",
1568 args.taskReductionArgs)))
1569 return parser.emitError(parser.getCurrentLocation())
1570 << "invalid `task_reduction` format";
1571
1572 if (failed(parseBlockArgClause(parser, entryBlockArgs, "use_device_addr",
1573 args.useDeviceAddrArgs)))
1574 return parser.emitError(parser.getCurrentLocation())
1575 << "invalid `use_device_addr` format";
1576
1577 if (failed(parseBlockArgClause(parser, entryBlockArgs, "use_device_ptr",
1578 args.useDevicePtrArgs)))
1579 return parser.emitError(parser.getCurrentLocation())
1580 << "invalid `use_device_addr` format";
1581
1582 return parser.parseRegion(region, entryBlockArgs);
1583}
1584
1585// These parseXyz functions correspond to the custom<Xyz> definitions
1586// in the .td file(s).
1587static ParseResult parseTargetOpRegion(
1588 OpAsmParser &parser, Region &region,
1590 SmallVectorImpl<Type> &hasDeviceAddrTypes,
1592 SmallVectorImpl<Type> &hostEvalTypes,
1594 SmallVectorImpl<Type> &mapTypes,
1596 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1597 UnitAttr &privateNeedsBarrier, DenseI64ArrayAttr &privateMaps) {
1598 AllRegionParseArgs args;
1599 args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
1600 args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
1601 args.mapArgs.emplace(mapVars, mapTypes);
1602 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1603 privateNeedsBarrier, &privateMaps);
1604 return parseBlockArgRegion(parser, region, args);
1605}
1606
1608 OpAsmParser &parser, Region &region,
1610 SmallVectorImpl<Type> &inReductionTypes,
1611 DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
1613 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1614 UnitAttr &privateNeedsBarrier) {
1615 AllRegionParseArgs args;
1616 args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
1617 inReductionByref, inReductionSyms);
1618 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1619 privateNeedsBarrier);
1620 return parseBlockArgRegion(parser, region, args);
1621}
1622
1624 OpAsmParser &parser, Region &region,
1626 SmallVectorImpl<Type> &inReductionTypes,
1627 DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
1629 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1630 UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods,
1632 SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
1633 ArrayAttr &reductionSyms) {
1634 AllRegionParseArgs args;
1635 args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
1636 inReductionByref, inReductionSyms);
1637 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1638 privateNeedsBarrier);
1639 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1640 reductionSyms, &reductionMods);
1641 return parseBlockArgRegion(parser, region, args);
1642}
1643
1644static ParseResult parsePrivateRegion(
1645 OpAsmParser &parser, Region &region,
1647 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1648 UnitAttr &privateNeedsBarrier) {
1649 AllRegionParseArgs args;
1650 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1651 privateNeedsBarrier);
1652 return parseBlockArgRegion(parser, region, args);
1653}
1654
1656 OpAsmParser &parser, Region &region,
1658 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1659 UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods,
1661 SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
1662 ArrayAttr &reductionSyms) {
1663 AllRegionParseArgs args;
1664 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1665 privateNeedsBarrier);
1666 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1667 reductionSyms, &reductionMods);
1668 return parseBlockArgRegion(parser, region, args);
1669}
1670
1672 OpAsmParser &parser, Region &region,
1674 SmallVectorImpl<Type> &linearTypes,
1676 SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearVarTypes,
1677 ArrayAttr &linearModifiers, ArrayAttr &linearSyms,
1679 llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
1680 UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods,
1682 SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
1683 ArrayAttr &reductionSyms) {
1684 AllRegionParseArgs args;
1685 args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
1686 linearStepTypes, linearVarTypes, linearModifiers,
1687 linearSyms);
1688 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1689 privateNeedsBarrier);
1690 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1691 reductionSyms, &reductionMods);
1692 return parseBlockArgRegion(parser, region, args);
1693}
1694
1695static ParseResult parseTaskReductionRegion(
1696 OpAsmParser &parser, Region &region,
1698 SmallVectorImpl<Type> &taskReductionTypes,
1699 DenseBoolArrayAttr &taskReductionByref, ArrayAttr &taskReductionSyms) {
1700 AllRegionParseArgs args;
1701 args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,
1702 taskReductionByref, taskReductionSyms);
1703 return parseBlockArgRegion(parser, region, args);
1704}
1705
1707 OpAsmParser &parser, Region &region,
1709 SmallVectorImpl<Type> &useDeviceAddrTypes,
1711 SmallVectorImpl<Type> &useDevicePtrTypes) {
1712 AllRegionParseArgs args;
1713 args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);
1714 args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);
1715 return parseBlockArgRegion(parser, region, args);
1716}
1717
1718//===----------------------------------------------------------------------===//
1719// Printers for operations including clauses that define entry block arguments.
1720//===----------------------------------------------------------------------===//
1721
1722namespace {
1723struct MapPrintArgs {
1724 ValueRange vars;
1725 TypeRange types;
1726 MapPrintArgs(ValueRange vars, TypeRange types) : vars(vars), types(types) {}
1727};
1728struct PrivatePrintArgs {
1729 ValueRange vars;
1730 TypeRange types;
1731 ArrayAttr syms;
1732 UnitAttr needsBarrier;
1733 DenseI64ArrayAttr mapIndices;
1734 PrivatePrintArgs(ValueRange vars, TypeRange types, ArrayAttr syms,
1735 UnitAttr needsBarrier, DenseI64ArrayAttr mapIndices)
1736 : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
1737 mapIndices(mapIndices) {}
1738};
1739struct LinearPrintArgs {
1740 ValueRange vars;
1741 TypeRange types;
1742 ValueRange stepVars;
1743 TypeRange stepTypes;
1744 ArrayAttr varTypes;
1745 ArrayAttr modifiers;
1746 ArrayAttr syms;
1747 LinearPrintArgs(ValueRange vars, TypeRange types, ValueRange stepVars,
1748 TypeRange stepTypes, ArrayAttr varTypes, ArrayAttr modifiers,
1749 ArrayAttr syms)
1750 : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
1751 varTypes(varTypes), modifiers(modifiers), syms(syms) {}
1752};
1753struct ReductionPrintArgs {
1754 ValueRange vars;
1755 TypeRange types;
1756 DenseBoolArrayAttr byref;
1757 ArrayAttr syms;
1758 ArrayAttr modifier;
1759 ReductionPrintArgs(ValueRange vars, TypeRange types, DenseBoolArrayAttr byref,
1760 ArrayAttr syms, ArrayAttr mod = nullptr)
1761 : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
1762};
1763struct AllRegionPrintArgs {
1764 std::optional<MapPrintArgs> hasDeviceAddrArgs;
1765 std::optional<MapPrintArgs> hostEvalArgs;
1766 std::optional<ReductionPrintArgs> inReductionArgs;
1767 std::optional<LinearPrintArgs> linearArgs;
1768 std::optional<MapPrintArgs> mapArgs;
1769 std::optional<PrivatePrintArgs> privateArgs;
1770 std::optional<ReductionPrintArgs> reductionArgs;
1771 std::optional<ReductionPrintArgs> taskReductionArgs;
1772 std::optional<MapPrintArgs> useDeviceAddrArgs;
1773 std::optional<MapPrintArgs> useDevicePtrArgs;
1774};
1775} // namespace
1776
1778 OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
1779 ValueRange argsSubrange, ValueRange operands, TypeRange types,
1780 ArrayAttr symbols = nullptr, DenseI64ArrayAttr mapIndices = nullptr,
1781 DenseBoolArrayAttr byref = nullptr, ArrayAttr modifier = nullptr,
1782 UnitAttr needsBarrier = nullptr) {
1783 if (argsSubrange.empty())
1784 return;
1785
1786 p << clauseName << "(";
1787
1788 if (!symbols) {
1789 llvm::SmallVector<Attribute> values(operands.size(), nullptr);
1790 symbols = ArrayAttr::get(ctx, values);
1791 }
1792
1793 if (!mapIndices) {
1794 llvm::SmallVector<int64_t> values(operands.size(), -1);
1795 mapIndices = DenseI64ArrayAttr::get(ctx, values);
1796 }
1797
1798 if (!byref) {
1799 mlir::SmallVector<bool> values(operands.size(), false);
1800 byref = DenseBoolArrayAttr::get(ctx, values);
1801 }
1802
1803 unsigned index = 0;
1804 llvm::interleaveComma(
1805 llvm::zip_equal(operands, argsSubrange, symbols, mapIndices.asArrayRef(),
1806 byref.asArrayRef()),
1807 p, [&](auto t) {
1808 auto [op, arg, sym, map, isByRef] = t;
1809 if (modifier) {
1810 ReductionModifier mod =
1811 cast<ReductionModifierAttr>(modifier[index]).getValue();
1812 if (mod != ReductionModifier::defaultmod)
1813 p << stringifyReductionModifier(mod) << " ";
1814 }
1815 ++index;
1816 if (isByRef)
1817 p << "byref ";
1818 if (sym)
1819 p << sym << " ";
1820 p << op << " -> " << arg;
1821 if (map != -1)
1822 p << " [map_idx=" << map << "]";
1823 });
1824 p << " : ";
1825 llvm::interleaveComma(types, p);
1826 p << ") ";
1827
1828 if (needsBarrier)
1829 p << getPrivateNeedsBarrierSpelling() << " ";
1830}
1831
1833 StringRef clauseName, ValueRange argsSubrange,
1834 std::optional<MapPrintArgs> mapArgs) {
1835 if (mapArgs)
1836 printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange, mapArgs->vars,
1837 mapArgs->types);
1838}
1839
1841 StringRef clauseName, ValueRange argsSubrange,
1842 std::optional<PrivatePrintArgs> privateArgs) {
1843 if (privateArgs)
1845 p, ctx, clauseName, argsSubrange, privateArgs->vars, privateArgs->types,
1846 privateArgs->syms, privateArgs->mapIndices, /*byref=*/nullptr,
1847 /*modifier=*/nullptr, privateArgs->needsBarrier);
1848}
1849
1850static void
1851printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
1852 ValueRange argsSubrange,
1853 std::optional<ReductionPrintArgs> reductionArgs) {
1854 if (reductionArgs)
1855 printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange,
1856 reductionArgs->vars, reductionArgs->types,
1857 reductionArgs->syms, /*mapIndices=*/nullptr,
1858 reductionArgs->byref, reductionArgs->modifier);
1859}
1860
1862 StringRef clauseName, ValueRange argsSubrange,
1863 std::optional<LinearPrintArgs> linearArgs) {
1864 if (!linearArgs || argsSubrange.empty())
1865 return;
1866
1867 p << clauseName << "(";
1868 for (auto [i, var] : llvm::enumerate(linearArgs->vars)) {
1869 if (i != 0)
1870 p << ", ";
1871 auto modifier =
1872 linearArgs->modifiers
1873 ? dyn_cast_if_present<LinearModifierAttr>(linearArgs->modifiers[i])
1874 : nullptr;
1875 if (modifier)
1876 p << stringifyLinearModifier(modifier.getValue()) << "(";
1877 if (linearArgs->syms)
1878 p << linearArgs->syms[i] << " ";
1879 p << var << " -> " << argsSubrange[i] << " : " << linearArgs->types[i]
1880 << " = " << linearArgs->stepVars[i] << " : " << linearArgs->stepTypes[i];
1881 if (modifier)
1882 p << ")";
1883 }
1884 p << ") ";
1885
1886 if (linearArgs->varTypes)
1887 p << "linear_var_types(" << linearArgs->varTypes << ") ";
1888}
1889
1891 const AllRegionPrintArgs &args) {
1892 auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op);
1893 MLIRContext *ctx = op->getContext();
1894
1895 printBlockArgClause(p, ctx, "has_device_addr",
1896 iface.getHasDeviceAddrBlockArgs(),
1897 args.hasDeviceAddrArgs);
1898 printBlockArgClause(p, ctx, "host_eval", iface.getHostEvalBlockArgs(),
1899 args.hostEvalArgs);
1900 printBlockArgClause(p, ctx, "in_reduction", iface.getInReductionBlockArgs(),
1901 args.inReductionArgs);
1902 printBlockArgClause(p, ctx, "linear", iface.getLinearBlockArgs(),
1903 args.linearArgs);
1904 printBlockArgClause(p, ctx, "map_entries", iface.getMapBlockArgs(),
1905 args.mapArgs);
1906 printBlockArgClause(p, ctx, "private", iface.getPrivateBlockArgs(),
1907 args.privateArgs);
1908 printBlockArgClause(p, ctx, "reduction", iface.getReductionBlockArgs(),
1909 args.reductionArgs);
1910 printBlockArgClause(p, ctx, "task_reduction",
1911 iface.getTaskReductionBlockArgs(),
1912 args.taskReductionArgs);
1913 printBlockArgClause(p, ctx, "use_device_addr",
1914 iface.getUseDeviceAddrBlockArgs(),
1915 args.useDeviceAddrArgs);
1916 printBlockArgClause(p, ctx, "use_device_ptr",
1917 iface.getUseDevicePtrBlockArgs(), args.useDevicePtrArgs);
1918
1919 p.printRegion(region, /*printEntryBlockArgs=*/false);
1920}
1921
1922// These parseXyz functions correspond to the custom<Xyz> definitions
1923// in the .td file(s).
1925 ValueRange hasDeviceAddrVars,
1926 TypeRange hasDeviceAddrTypes,
1927 ValueRange hostEvalVars,
1928 TypeRange hostEvalTypes, ValueRange mapVars,
1929 TypeRange mapTypes, ValueRange privateVars,
1930 TypeRange privateTypes, ArrayAttr privateSyms,
1931 UnitAttr privateNeedsBarrier,
1932 DenseI64ArrayAttr privateMaps) {
1933 AllRegionPrintArgs args;
1934 args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
1935 args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
1936 args.mapArgs.emplace(mapVars, mapTypes);
1937 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1938 privateNeedsBarrier, privateMaps);
1939 printBlockArgRegion(p, op, region, args);
1940}
1941
1943 OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
1944 TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
1945 ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
1946 ArrayAttr privateSyms, UnitAttr privateNeedsBarrier) {
1947 AllRegionPrintArgs args;
1948 args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
1949 inReductionByref, inReductionSyms);
1950 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1951 privateNeedsBarrier,
1952 /*mapIndices=*/nullptr);
1953 printBlockArgRegion(p, op, region, args);
1954}
1955
1957 OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
1958 TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
1959 ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
1960 ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
1961 ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes,
1962 DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
1963 AllRegionPrintArgs args;
1964 args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
1965 inReductionByref, inReductionSyms);
1966 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1967 privateNeedsBarrier,
1968 /*mapIndices=*/nullptr);
1969 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1970 reductionSyms, reductionMods);
1971 printBlockArgRegion(p, op, region, args);
1972}
1973
1975 ValueRange privateVars, TypeRange privateTypes,
1976 ArrayAttr privateSyms,
1977 UnitAttr privateNeedsBarrier) {
1978 AllRegionPrintArgs args;
1979 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1980 privateNeedsBarrier,
1981 /*mapIndices=*/nullptr);
1982 printBlockArgRegion(p, op, region, args);
1983}
1984
1986 OpAsmPrinter &p, Operation *op, Region &region, ValueRange privateVars,
1987 TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
1988 ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes,
1989 DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
1990 AllRegionPrintArgs args;
1991 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
1992 privateNeedsBarrier,
1993 /*mapIndices=*/nullptr);
1994 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
1995 reductionSyms, reductionMods);
1996 printBlockArgRegion(p, op, region, args);
1997}
1998
2000 OpAsmPrinter &p, Operation *op, Region &region, ValueRange linearVars,
2001 TypeRange linearTypes, ValueRange linearStepVars, TypeRange linearStepTypes,
2002 ArrayAttr linearVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms,
2003 ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms,
2004 UnitAttr privateNeedsBarrier, ArrayAttr reductionMods,
2005 ValueRange reductionVars, TypeRange reductionTypes,
2006 DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
2007 AllRegionPrintArgs args;
2008 args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
2009 linearStepTypes, linearVarTypes, linearModifiers,
2010 linearSyms);
2011 args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
2012 privateNeedsBarrier,
2013 /*mapIndices=*/nullptr);
2014 args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
2015 reductionSyms, reductionMods);
2016 printBlockArgRegion(p, op, region, args);
2017}
2018
2020 Region &region,
2021 ValueRange taskReductionVars,
2022 TypeRange taskReductionTypes,
2023 DenseBoolArrayAttr taskReductionByref,
2024 ArrayAttr taskReductionSyms) {
2025 AllRegionPrintArgs args;
2026 args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,
2027 taskReductionByref, taskReductionSyms);
2028 printBlockArgRegion(p, op, region, args);
2029}
2030
2032 Region &region,
2033 ValueRange useDeviceAddrVars,
2034 TypeRange useDeviceAddrTypes,
2035 ValueRange useDevicePtrVars,
2036 TypeRange useDevicePtrTypes) {
2037 AllRegionPrintArgs args;
2038 args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);
2039 args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);
2040 printBlockArgRegion(p, op, region, args);
2041}
2042
2043template <typename ParsePrefixFn>
2044static ParseResult parseSplitIteratedList(
2045 OpAsmParser &parser,
2047 SmallVectorImpl<Type> &iteratedTypes,
2049 SmallVectorImpl<Type> &plainTypes, ParsePrefixFn &&parsePrefix) {
2050
2051 return parser.parseCommaSeparatedList([&]() -> ParseResult {
2052 if (failed(parsePrefix()))
2053 return failure();
2054
2056 Type ty;
2057 if (parser.parseOperand(v) || parser.parseColonType(ty))
2058 return failure();
2059
2060 if (llvm::isa<mlir::omp::IteratedType>(ty)) {
2061 iteratedVars.push_back(v);
2062 iteratedTypes.push_back(ty);
2063 } else {
2064 plainVars.push_back(v);
2065 plainTypes.push_back(ty);
2066 }
2067 return success();
2068 });
2069}
2070
2071template <typename PrintPrefixFn>
2073 TypeRange iteratedTypes,
2074 ValueRange plainVars, TypeRange plainTypes,
2075 PrintPrefixFn &&printPrefixForPlain,
2076 PrintPrefixFn &&printPrefixForIterated) {
2077
2078 bool first = true;
2079 auto emit = [&](Value v, Type t, auto &&printPrefix) {
2080 if (!first)
2081 p << ", ";
2082 printPrefix(v, t);
2083 p << v << " : " << t;
2084 first = false;
2085 };
2086
2087 for (unsigned i = 0; i < iteratedVars.size(); ++i)
2088 emit(iteratedVars[i], iteratedTypes[i], printPrefixForIterated);
2089 for (unsigned i = 0; i < plainVars.size(); ++i)
2090 emit(plainVars[i], plainTypes[i], printPrefixForPlain);
2091}
2092
2093/// Verify the per-variable modifier list has one entry per reduction variable.
2094static LogicalResult verifyReductionModifiers(Operation *op,
2095 OperandRange reductionVars,
2096 ArrayAttr reductionMods) {
2097 if (reductionMods && reductionMods.size() != reductionVars.size())
2098 return op->emitOpError(
2099 "expected as many reduction modifiers as reduction variables");
2100 return success();
2101}
2102
2103/// Verifies Reduction Clause
2104static LogicalResult
2105verifyReductionVarList(Operation *op, std::optional<ArrayAttr> reductionSyms,
2106 OperandRange reductionVars,
2107 std::optional<ArrayRef<bool>> reductionByref,
2108 ArrayAttr reductionMods = nullptr) {
2109 if (failed(verifyReductionModifiers(op, reductionVars, reductionMods)))
2110 return failure();
2111 if (!reductionVars.empty()) {
2112 if (!reductionSyms || reductionSyms->size() != reductionVars.size())
2113 return op->emitOpError()
2114 << "expected as many reduction symbol references "
2115 "as reduction variables";
2116 if (reductionByref && reductionByref->size() != reductionVars.size())
2117 return op->emitError() << "expected as many reduction variable by "
2118 "reference attributes as reduction variables";
2119 } else {
2120 if (reductionSyms)
2121 return op->emitOpError() << "unexpected reduction symbol references";
2122 return success();
2123 }
2124
2125 // TODO: The followings should be done in
2126 // SymbolUserOpInterface::verifySymbolUses.
2127 DenseSet<Value> accumulators;
2128 for (auto args : llvm::zip(reductionVars, *reductionSyms)) {
2129 Value accum = std::get<0>(args);
2130
2131 if (!accumulators.insert(accum).second)
2132 return op->emitOpError() << "accumulator variable used more than once";
2133
2134 Type varType = accum.getType();
2135 auto symbolRef = llvm::cast<SymbolRefAttr>(std::get<1>(args));
2136 auto decl =
2138 if (!decl)
2139 return op->emitOpError() << "expected symbol reference " << symbolRef
2140 << " to point to a reduction declaration";
2141
2142 if (decl.getAccumulatorType() && decl.getAccumulatorType() != varType)
2143 return op->emitOpError()
2144 << "expected accumulator (" << varType
2145 << ") to be the same type as reduction declaration ("
2146 << decl.getAccumulatorType() << ")";
2147 }
2148
2149 return success();
2150}
2151
2152//===----------------------------------------------------------------------===//
2153// Parser, printer and verifier for Copyprivate
2154//===----------------------------------------------------------------------===//
2155
2156/// copyprivate-entry-list ::= copyprivate-entry
2157/// | copyprivate-entry-list `,` copyprivate-entry
2158/// copyprivate-entry ::= ssa-id `->` symbol-ref `:` type
2159static ParseResult parseCopyprivate(
2160 OpAsmParser &parser,
2162 SmallVectorImpl<Type> &copyprivateTypes, ArrayAttr &copyprivateSyms) {
2164 if (failed(parser.parseCommaSeparatedList([&]() {
2165 if (parser.parseOperand(copyprivateVars.emplace_back()) ||
2166 parser.parseArrow() ||
2167 parser.parseAttribute(symsVec.emplace_back()) ||
2168 parser.parseColonType(copyprivateTypes.emplace_back()))
2169 return failure();
2170 return success();
2171 })))
2172 return failure();
2173 SmallVector<Attribute> syms(symsVec.begin(), symsVec.end());
2174 copyprivateSyms = ArrayAttr::get(parser.getContext(), syms);
2175 return success();
2176}
2177
2178/// Print Copyprivate clause
2180 OperandRange copyprivateVars,
2181 TypeRange copyprivateTypes,
2182 std::optional<ArrayAttr> copyprivateSyms) {
2183 if (!copyprivateSyms.has_value())
2184 return;
2185 llvm::interleaveComma(
2186 llvm::zip(copyprivateVars, *copyprivateSyms, copyprivateTypes), p,
2187 [&](const auto &args) {
2188 p << std::get<0>(args) << " -> " << std::get<1>(args) << " : "
2189 << std::get<2>(args);
2190 });
2191}
2192
2193/// Verifies CopyPrivate Clause
2194static LogicalResult
2196 std::optional<ArrayAttr> copyprivateSyms) {
2197 size_t copyprivateSymsSize =
2198 copyprivateSyms.has_value() ? copyprivateSyms->size() : 0;
2199 if (copyprivateSymsSize != copyprivateVars.size())
2200 return op->emitOpError() << "inconsistent number of copyprivate vars (= "
2201 << copyprivateVars.size()
2202 << ") and functions (= " << copyprivateSymsSize
2203 << "), both must be equal";
2204 if (!copyprivateSyms.has_value())
2205 return success();
2206
2207 for (auto copyprivateVarAndSym :
2208 llvm::zip(copyprivateVars, *copyprivateSyms)) {
2209 auto symbolRef =
2210 llvm::cast<SymbolRefAttr>(std::get<1>(copyprivateVarAndSym));
2211 std::optional<std::variant<mlir::func::FuncOp, mlir::LLVM::LLVMFuncOp>>
2212 funcOp;
2213 if (mlir::func::FuncOp mlirFuncOp =
2215 symbolRef))
2216 funcOp = mlirFuncOp;
2217 else if (mlir::LLVM::LLVMFuncOp llvmFuncOp =
2219 op, symbolRef))
2220 funcOp = llvmFuncOp;
2221
2222 auto getNumArguments = [&] {
2223 return std::visit([](auto &f) { return f.getNumArguments(); }, *funcOp);
2224 };
2225
2226 auto getArgumentType = [&](unsigned i) {
2227 return std::visit([i](auto &f) { return f.getArgumentTypes()[i]; },
2228 *funcOp);
2229 };
2230
2231 if (!funcOp)
2232 return op->emitOpError() << "expected symbol reference " << symbolRef
2233 << " to point to a copy function";
2234
2235 if (getNumArguments() != 2)
2236 return op->emitOpError()
2237 << "expected copy function " << symbolRef << " to have 2 operands";
2238
2239 Type argTy = getArgumentType(0);
2240 if (argTy != getArgumentType(1))
2241 return op->emitOpError() << "expected copy function " << symbolRef
2242 << " arguments to have the same type";
2243
2244 Type varType = std::get<0>(copyprivateVarAndSym).getType();
2245 if (argTy != varType)
2246 return op->emitOpError()
2247 << "expected copy function arguments' type (" << argTy
2248 << ") to be the same as copyprivate variable's type (" << varType
2249 << ")";
2250 }
2251
2252 return success();
2253}
2254
2255//===----------------------------------------------------------------------===//
2256// Parser, printer and verifier for DependVarList
2257//===----------------------------------------------------------------------===//
2258
2259/// depend-entry-list ::= depend-entry
2260/// | depend-entry-list `,` depend-entry
2261/// depend-entry ::= depend-kind `->` ssa-id `:` type
2262/// | depend-kind `->` ssa-id `:` iterated-type
2263static ParseResult parseDependVarList(
2264 OpAsmParser &parser,
2266 SmallVectorImpl<Type> &dependTypes, ArrayAttr &dependKinds,
2268 SmallVectorImpl<Type> &iteratedTypes, ArrayAttr &iteratedKinds) {
2271 if (failed(parser.parseCommaSeparatedList([&]() {
2272 StringRef keyword;
2273 OpAsmParser::UnresolvedOperand operand;
2274 Type ty;
2275 if (parser.parseKeyword(&keyword) || parser.parseArrow() ||
2276 parser.parseOperand(operand) || parser.parseColonType(ty))
2277 return failure();
2278 std::optional<ClauseTaskDepend> keywordDepend =
2279 symbolizeClauseTaskDepend(keyword);
2280 if (!keywordDepend)
2281 return failure();
2282 auto kindAttr =
2283 ClauseTaskDependAttr::get(parser.getContext(), *keywordDepend);
2284 if (llvm::isa<mlir::omp::IteratedType>(ty)) {
2285 iteratedVars.push_back(operand);
2286 iteratedTypes.push_back(ty);
2287 iterKindsVec.push_back(kindAttr);
2288 } else {
2289 dependVars.push_back(operand);
2290 dependTypes.push_back(ty);
2291 kindsVec.push_back(kindAttr);
2292 }
2293 return success();
2294 })))
2295 return failure();
2296 SmallVector<Attribute> kinds(kindsVec.begin(), kindsVec.end());
2297 dependKinds = ArrayAttr::get(parser.getContext(), kinds);
2298 SmallVector<Attribute> iterKinds(iterKindsVec.begin(), iterKindsVec.end());
2299 iteratedKinds = ArrayAttr::get(parser.getContext(), iterKinds);
2300 return success();
2301}
2302
2303/// Print Depend clause
2305 OperandRange dependVars, TypeRange dependTypes,
2306 std::optional<ArrayAttr> dependKinds,
2307 OperandRange iteratedVars,
2308 TypeRange iteratedTypes,
2309 std::optional<ArrayAttr> iteratedKinds) {
2310 bool first = true;
2311 auto printEntries = [&](OperandRange vars, TypeRange types,
2312 std::optional<ArrayAttr> kinds) {
2313 for (unsigned i = 0, e = vars.size(); i < e; ++i) {
2314 if (!first)
2315 p << ", ";
2316 p << stringifyClauseTaskDepend(
2317 llvm::cast<mlir::omp::ClauseTaskDependAttr>((*kinds)[i])
2318 .getValue())
2319 << " -> " << vars[i] << " : " << types[i];
2320 first = false;
2321 }
2322 };
2323 printEntries(dependVars, dependTypes, dependKinds);
2324 printEntries(iteratedVars, iteratedTypes, iteratedKinds);
2325}
2326
2327/// Verifies Depend clause
2328static LogicalResult verifyDependVarList(Operation *op,
2329 std::optional<ArrayAttr> dependKinds,
2330 OperandRange dependVars,
2331 std::optional<ArrayAttr> iteratedKinds,
2332 OperandRange iteratedVars) {
2333 if (!dependVars.empty()) {
2334 if (!dependKinds || dependKinds->size() != dependVars.size())
2335 return op->emitOpError() << "expected as many depend values"
2336 " as depend variables";
2337 } else {
2338 if (dependKinds && !dependKinds->empty())
2339 return op->emitOpError() << "unexpected depend values";
2340 }
2341
2342 if (!iteratedVars.empty()) {
2343 if (!iteratedKinds || iteratedKinds->size() != iteratedVars.size())
2344 return op->emitOpError() << "expected as many depend iterated values"
2345 " as depend iterated variables";
2346 } else {
2347 if (iteratedKinds && !iteratedKinds->empty())
2348 return op->emitOpError() << "unexpected depend iterated values";
2349 }
2350
2351 return success();
2352}
2353
2354//===----------------------------------------------------------------------===//
2355// Parser, printer and verifier for Synchronization Hint (2.17.12)
2356//===----------------------------------------------------------------------===//
2357
2358/// Parses a Synchronization Hint clause. The value of hint is an integer
2359/// which is a combination of different hints from `omp_sync_hint_t`.
2360///
2361/// hint-clause = `hint` `(` hint-value `)`
2362static ParseResult parseSynchronizationHint(OpAsmParser &parser,
2363 IntegerAttr &hintAttr) {
2364 StringRef hintKeyword;
2365 int64_t hint = 0;
2366 if (succeeded(parser.parseOptionalKeyword("none"))) {
2367 hintAttr = IntegerAttr::get(parser.getBuilder().getI64Type(), 0);
2368 return success();
2369 }
2370 auto parseKeyword = [&]() -> ParseResult {
2371 if (failed(parser.parseKeyword(&hintKeyword)))
2372 return failure();
2373 if (hintKeyword == "uncontended")
2374 hint |= 1;
2375 else if (hintKeyword == "contended")
2376 hint |= 2;
2377 else if (hintKeyword == "nonspeculative")
2378 hint |= 4;
2379 else if (hintKeyword == "speculative")
2380 hint |= 8;
2381 else
2382 return parser.emitError(parser.getCurrentLocation())
2383 << hintKeyword << " is not a valid hint";
2384 return success();
2385 };
2386 if (parser.parseCommaSeparatedList(parseKeyword))
2387 return failure();
2388 hintAttr = IntegerAttr::get(parser.getBuilder().getI64Type(), hint);
2389 return success();
2390}
2391
2392/// Prints a Synchronization Hint clause
2394 IntegerAttr hintAttr) {
2395 int64_t hint = hintAttr.getInt();
2396
2397 if (hint == 0) {
2398 p << "none";
2399 return;
2400 }
2401
2402 // Helper function to get n-th bit from the right end of `value`
2403 auto bitn = [](int value, int n) -> bool { return value & (1 << n); };
2404
2405 bool uncontended = bitn(hint, 0);
2406 bool contended = bitn(hint, 1);
2407 bool nonspeculative = bitn(hint, 2);
2408 bool speculative = bitn(hint, 3);
2409
2411 if (uncontended)
2412 hints.push_back("uncontended");
2413 if (contended)
2414 hints.push_back("contended");
2415 if (nonspeculative)
2416 hints.push_back("nonspeculative");
2417 if (speculative)
2418 hints.push_back("speculative");
2419
2420 llvm::interleaveComma(hints, p);
2421}
2422
2423/// Verifies a synchronization hint clause
2424static LogicalResult verifySynchronizationHint(Operation *op, uint64_t hint) {
2425
2426 // Helper function to get n-th bit from the right end of `value`
2427 auto bitn = [](int value, int n) -> bool { return value & (1 << n); };
2428
2429 bool uncontended = bitn(hint, 0);
2430 bool contended = bitn(hint, 1);
2431 bool nonspeculative = bitn(hint, 2);
2432 bool speculative = bitn(hint, 3);
2433
2434 if (uncontended && contended)
2435 return op->emitOpError() << "the hints omp_sync_hint_uncontended and "
2436 "omp_sync_hint_contended cannot be combined";
2437 if (nonspeculative && speculative)
2438 return op->emitOpError() << "the hints omp_sync_hint_nonspeculative and "
2439 "omp_sync_hint_speculative cannot be combined.";
2440 return success();
2441}
2442
2443//===----------------------------------------------------------------------===//
2444// Parser, printer and verifier for Target
2445//===----------------------------------------------------------------------===//
2446
2447// Helper function to get bitwise AND of `value` and 'flag' then return it as a
2448// boolean
2449static bool mapTypeToBool(ClauseMapFlags value, ClauseMapFlags flag) {
2450 return (value & flag) == flag;
2451}
2452
2453/// Parses a map_entries map type from a string format back into its numeric
2454/// value.
2455///
2456/// map-clause = `map_clauses ( ( `(` `always, `? `implicit, `? `ompx_hold, `?
2457/// `close, `? `present, `? ( `to` | `from` | `delete` `)` )+ `)` )
2458static ParseResult parseMapClause(OpAsmParser &parser,
2459 ClauseMapFlagsAttr &mapType) {
2460 ClauseMapFlags mapTypeBits = ClauseMapFlags::none;
2461 // This simply verifies the correct keyword is read in, the
2462 // keyword itself is stored inside of the operation
2463 auto parseTypeAndMod = [&]() -> ParseResult {
2464 StringRef mapTypeMod;
2465 if (parser.parseKeyword(&mapTypeMod))
2466 return failure();
2467
2468 if (mapTypeMod == "always")
2469 mapTypeBits |= ClauseMapFlags::always;
2470
2471 if (mapTypeMod == "implicit")
2472 mapTypeBits |= ClauseMapFlags::implicit;
2473
2474 if (mapTypeMod == "ompx_hold")
2475 mapTypeBits |= ClauseMapFlags::ompx_hold;
2476
2477 if (mapTypeMod == "close")
2478 mapTypeBits |= ClauseMapFlags::close;
2479
2480 if (mapTypeMod == "present")
2481 mapTypeBits |= ClauseMapFlags::present;
2482
2483 if (mapTypeMod == "to")
2484 mapTypeBits |= ClauseMapFlags::to;
2485
2486 if (mapTypeMod == "from")
2487 mapTypeBits |= ClauseMapFlags::from;
2488
2489 if (mapTypeMod == "tofrom")
2490 mapTypeBits |= ClauseMapFlags::to | ClauseMapFlags::from;
2491
2492 if (mapTypeMod == "delete")
2493 mapTypeBits |= ClauseMapFlags::del;
2494
2495 if (mapTypeMod == "storage")
2496 mapTypeBits |= ClauseMapFlags::storage;
2497
2498 if (mapTypeMod == "return_param")
2499 mapTypeBits |= ClauseMapFlags::return_param;
2500
2501 if (mapTypeMod == "private")
2502 mapTypeBits |= ClauseMapFlags::priv;
2503
2504 if (mapTypeMod == "literal")
2505 mapTypeBits |= ClauseMapFlags::literal;
2506
2507 if (mapTypeMod == "attach")
2508 mapTypeBits |= ClauseMapFlags::attach;
2509
2510 if (mapTypeMod == "attach_always")
2511 mapTypeBits |= ClauseMapFlags::attach_always;
2512
2513 if (mapTypeMod == "attach_never")
2514 mapTypeBits |= ClauseMapFlags::attach_never;
2515
2516 if (mapTypeMod == "attach_auto")
2517 mapTypeBits |= ClauseMapFlags::attach_auto;
2518
2519 if (mapTypeMod == "ref_ptr")
2520 mapTypeBits |= ClauseMapFlags::ref_ptr;
2521
2522 if (mapTypeMod == "ref_ptee")
2523 mapTypeBits |= ClauseMapFlags::ref_ptee;
2524
2525 if (mapTypeMod == "is_device_ptr")
2526 mapTypeBits |= ClauseMapFlags::is_device_ptr;
2527
2528 if (mapTypeMod == "target_param")
2529 mapTypeBits |= ClauseMapFlags::target_param;
2530
2531 return success();
2532 };
2533
2534 if (parser.parseCommaSeparatedList(parseTypeAndMod))
2535 return failure();
2536
2537 mapType =
2538 parser.getBuilder().getAttr<mlir::omp::ClauseMapFlagsAttr>(mapTypeBits);
2539
2540 return success();
2541}
2542
2543/// Prints a map_entries map type from its numeric value out into its string
2544/// format.
2545static void printMapClause(OpAsmPrinter &p, Operation *op,
2546 ClauseMapFlagsAttr mapType) {
2548 ClauseMapFlags mapFlags = mapType.getValue();
2549
2550 // handling of always, close, present placed at the beginning of the string
2551 // to aid readability
2552 if (mapTypeToBool(mapFlags, ClauseMapFlags::always))
2553 mapTypeStrs.push_back("always");
2554 if (mapTypeToBool(mapFlags, ClauseMapFlags::implicit))
2555 mapTypeStrs.push_back("implicit");
2556 if (mapTypeToBool(mapFlags, ClauseMapFlags::ompx_hold))
2557 mapTypeStrs.push_back("ompx_hold");
2558 if (mapTypeToBool(mapFlags, ClauseMapFlags::close))
2559 mapTypeStrs.push_back("close");
2560 if (mapTypeToBool(mapFlags, ClauseMapFlags::present))
2561 mapTypeStrs.push_back("present");
2562 if (mapTypeToBool(mapFlags, ClauseMapFlags::target_param))
2563 mapTypeStrs.push_back("target_param");
2564
2565 // special handling of to/from/tofrom/delete and release/alloc, release +
2566 // alloc are the abscense of one of the other flags, whereas tofrom requires
2567 // both the to and from flag to be set.
2568 bool to = mapTypeToBool(mapFlags, ClauseMapFlags::to);
2569 bool from = mapTypeToBool(mapFlags, ClauseMapFlags::from);
2570
2571 if (to && from)
2572 mapTypeStrs.push_back("tofrom");
2573 else if (from)
2574 mapTypeStrs.push_back("from");
2575 else if (to)
2576 mapTypeStrs.push_back("to");
2577
2578 if (mapTypeToBool(mapFlags, ClauseMapFlags::del))
2579 mapTypeStrs.push_back("delete");
2580 if (mapTypeToBool(mapFlags, ClauseMapFlags::return_param))
2581 mapTypeStrs.push_back("return_param");
2582 if (mapTypeToBool(mapFlags, ClauseMapFlags::storage))
2583 mapTypeStrs.push_back("storage");
2584 if (mapTypeToBool(mapFlags, ClauseMapFlags::priv))
2585 mapTypeStrs.push_back("private");
2586 if (mapTypeToBool(mapFlags, ClauseMapFlags::literal))
2587 mapTypeStrs.push_back("literal");
2588 if (mapTypeToBool(mapFlags, ClauseMapFlags::attach))
2589 mapTypeStrs.push_back("attach");
2590 if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_always))
2591 mapTypeStrs.push_back("attach_always");
2592 if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_never))
2593 mapTypeStrs.push_back("attach_never");
2594 if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_auto))
2595 mapTypeStrs.push_back("attach_auto");
2596 if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptr))
2597 mapTypeStrs.push_back("ref_ptr");
2598 if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptee))
2599 mapTypeStrs.push_back("ref_ptee");
2600 if (mapTypeToBool(mapFlags, ClauseMapFlags::is_device_ptr))
2601 mapTypeStrs.push_back("is_device_ptr");
2602 if (mapFlags == ClauseMapFlags::none)
2603 mapTypeStrs.push_back("none");
2604
2605 for (unsigned int i = 0; i < mapTypeStrs.size(); ++i) {
2606 p << mapTypeStrs[i];
2607 if (i + 1 < mapTypeStrs.size()) {
2608 p << ", ";
2609 }
2610 }
2611}
2612
2613static ParseResult parseMembersIndex(OpAsmParser &parser,
2614 ArrayAttr &membersIdx) {
2615 SmallVector<Attribute> values, memberIdxs;
2616
2617 auto parseIndices = [&]() -> ParseResult {
2618 int64_t value;
2619 if (parser.parseInteger(value))
2620 return failure();
2621 values.push_back(IntegerAttr::get(parser.getBuilder().getIntegerType(64),
2622 APInt(64, value, /*isSigned=*/false)));
2623 return success();
2624 };
2625
2626 do {
2627 if (failed(parser.parseLSquare()))
2628 return failure();
2629
2630 if (parser.parseCommaSeparatedList(parseIndices))
2631 return failure();
2632
2633 if (failed(parser.parseRSquare()))
2634 return failure();
2635
2636 memberIdxs.push_back(ArrayAttr::get(parser.getContext(), values));
2637 values.clear();
2638 } while (succeeded(parser.parseOptionalComma()));
2639
2640 if (!memberIdxs.empty())
2641 membersIdx = ArrayAttr::get(parser.getContext(), memberIdxs);
2642
2643 return success();
2644}
2645
2646static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op,
2647 ArrayAttr membersIdx) {
2648 if (!membersIdx)
2649 return;
2650
2651 llvm::interleaveComma(membersIdx, p, [&p](Attribute v) {
2652 p << "[";
2653 auto memberIdx = cast<ArrayAttr>(v);
2654 llvm::interleaveComma(memberIdx.getValue(), p, [&p](Attribute v2) {
2655 p << cast<IntegerAttr>(v2).getInt();
2656 });
2657 p << "]";
2658 });
2659}
2660
2662 VariableCaptureKindAttr mapCaptureType) {
2663 std::string typeCapStr;
2664 llvm::raw_string_ostream typeCap(typeCapStr);
2665 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByRef)
2666 typeCap << "ByRef";
2667 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByCopy)
2668 typeCap << "ByCopy";
2669 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::VLAType)
2670 typeCap << "VLAType";
2671 if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::This)
2672 typeCap << "This";
2673 p << typeCapStr;
2674}
2675
2676static ParseResult parseCaptureType(OpAsmParser &parser,
2677 VariableCaptureKindAttr &mapCaptureType) {
2678 StringRef mapCaptureKey;
2679 if (parser.parseKeyword(&mapCaptureKey))
2680 return failure();
2681
2682 if (mapCaptureKey == "This")
2683 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2684 parser.getContext(), mlir::omp::VariableCaptureKind::This);
2685 if (mapCaptureKey == "ByRef")
2686 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2687 parser.getContext(), mlir::omp::VariableCaptureKind::ByRef);
2688 if (mapCaptureKey == "ByCopy")
2689 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2690 parser.getContext(), mlir::omp::VariableCaptureKind::ByCopy);
2691 if (mapCaptureKey == "VLAType")
2692 mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
2693 parser.getContext(), mlir::omp::VariableCaptureKind::VLAType);
2694
2695 return success();
2696}
2697
2698static LogicalResult verifyMapInfoForMapClause(
2699 Operation *op, mlir::omp::MapInfoOp mapInfoOp,
2702 &updateFromVars) {
2703 mlir::omp::ClauseMapFlags mapTypeBits = mapInfoOp.getMapType();
2704
2705 bool to = mapTypeToBool(mapTypeBits, ClauseMapFlags::to);
2706 bool from = mapTypeToBool(mapTypeBits, ClauseMapFlags::from);
2707 bool del = mapTypeToBool(mapTypeBits, ClauseMapFlags::del);
2708
2709 bool always = mapTypeToBool(mapTypeBits, ClauseMapFlags::always);
2710 bool close = mapTypeToBool(mapTypeBits, ClauseMapFlags::close);
2711 bool implicit = mapTypeToBool(mapTypeBits, ClauseMapFlags::implicit);
2712 bool attach = mapTypeToBool(mapTypeBits, ClauseMapFlags::attach);
2713
2714 if ((isa<TargetDataOp>(op) || isa<TargetOp>(op)) && del)
2715 return emitError(op->getLoc(),
2716 "to, from, tofrom and alloc map types are permitted");
2717
2718 if (isa<TargetEnterDataOp>(op) && (from || del))
2719 return emitError(op->getLoc(), "to and alloc map types are permitted");
2720
2721 if (isa<TargetExitDataOp>(op) && to)
2722 return emitError(op->getLoc(),
2723 "from, release and delete map types are permitted");
2724
2725 if (isa<TargetUpdateOp>(op)) {
2726 if (del) {
2727 return emitError(op->getLoc(),
2728 "at least one of to or from map types must be "
2729 "specified, other map types are not permitted");
2730 }
2731
2732 if (!to && !from && !attach) {
2733 return emitError(op->getLoc(),
2734 "at least one of to or from or attach map types must be "
2735 "specified, other map types are not permitted");
2736 }
2737
2738 auto updateVar = mapInfoOp.getVarPtr();
2739
2740 if ((to && from) || (to && updateFromVars.contains(updateVar)) ||
2741 (from && updateToVars.contains(updateVar))) {
2742 return emitError(
2743 op->getLoc(),
2744 "either to or from map types can be specified, not both");
2745 }
2746
2747 if (always || close || implicit) {
2748 return emitError(
2749 op->getLoc(),
2750 "present, mapper and iterator map type modifiers are permitted");
2751 }
2752
2753 // It's possible we have an attach map, in which case if there is no to
2754 // or from tied to it, we skip insertion.
2755 if (to || from) {
2756 to ? updateToVars.insert(updateVar) : updateFromVars.insert(updateVar);
2757 }
2758 }
2759
2760 if ((mapInfoOp.getVarPtrPtr() && !mapInfoOp.getVarPtrPtrType()) ||
2761 (!mapInfoOp.getVarPtrPtr() && mapInfoOp.getVarPtrPtrType())) {
2762 return emitError(op->getLoc(),
2763 "if varPtrPtr or varPtrPtrType is specified, then both "
2764 "must be present");
2765 }
2766
2767 return success();
2768}
2769
2770static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars,
2771 OperandRange mapIterated) {
2774
2775 for (auto mapOp : mapVars) {
2776 if (!mapOp.getDefiningOp())
2777 return emitError(op->getLoc(), "missing map operation");
2778
2779 if (auto mapInfoOp = mapOp.getDefiningOp<mlir::omp::MapInfoOp>()) {
2780 if (failed(verifyMapInfoForMapClause(op, mapInfoOp, updateToVars,
2781 updateFromVars)))
2782 return failure();
2783 } else if (!isa<DeclareMapperInfoOp>(op)) {
2784 return emitError(op->getLoc(),
2785 "map argument is not a map entry operation");
2786 }
2787 }
2788
2789 // Verify iterated map entries.
2790 for (auto iterVal : mapIterated) {
2791 auto iterOp = iterVal.getDefiningOp<mlir::omp::IteratorOp>();
2792 if (!iterOp)
2793 return op->emitOpError() << "'map_iterated' arguments must be defined by "
2794 "'omp.iterator' ops";
2795
2796 // Check that the iterator body yields a value defined by omp.map.info.
2797 mlir::omp::YieldOp yieldOp = iterOp.getYieldOp();
2798 auto yieldedMapInfo =
2799 yieldOp && yieldOp.getNumOperands() == 1
2800 ? yieldOp.getResults()[0].getDefiningOp<mlir::omp::MapInfoOp>()
2801 : nullptr;
2802 if (!yieldedMapInfo)
2803 return op->emitOpError() << "'map_iterated' iterator body must yield "
2804 "a value defined by 'omp.map.info'";
2805
2806 if (failed(verifyMapInfoForMapClause(op, yieldedMapInfo, updateToVars,
2807 updateFromVars)))
2808 return failure();
2809 }
2810
2811 return success();
2812}
2813
2814template <typename OpType>
2815static LogicalResult verifyPrivateVarList(OpType &op);
2816
2817static LogicalResult verifyPrivateVarsMapping(TargetOp targetOp) {
2818 std::optional<DenseI64ArrayAttr> privateMapIndices =
2819 targetOp.getPrivateMapsAttr();
2820
2821 // None of the private operands are mapped.
2822 if (!privateMapIndices.has_value() || !privateMapIndices.value())
2823 return success();
2824
2825 OperandRange privateVars = targetOp.getPrivateVars();
2826
2827 if (privateMapIndices.value().size() !=
2828 static_cast<int64_t>(privateVars.size()))
2829 return emitError(targetOp.getLoc(), "sizes of `private` operand range and "
2830 "`private_maps` attribute mismatch");
2831
2832 return success();
2833}
2834
2835//===----------------------------------------------------------------------===//
2836// MapInfoOp
2837//===----------------------------------------------------------------------===//
2838
2839static LogicalResult verifyMapInfoDefinedArgs(Operation *op,
2840 StringRef clauseName,
2841 OperandRange vars) {
2842 for (Value var : vars)
2843 if (!llvm::isa_and_present<MapInfoOp>(var.getDefiningOp()))
2844 return op->emitOpError()
2845 << "'" << clauseName
2846 << "' arguments must be defined by 'omp.map.info' ops";
2847 return success();
2848}
2849
2850LogicalResult MapInfoOp::verify() {
2851 if (getMapperId() &&
2853 *this, getMapperIdAttr())) {
2854 return emitError("invalid mapper id");
2855 }
2856
2857 if (failed(verifyMapInfoDefinedArgs(*this, "members", getMembers())))
2858 return failure();
2859
2860 return success();
2861}
2862
2863//===----------------------------------------------------------------------===//
2864// TargetDataOp
2865//===----------------------------------------------------------------------===//
2866
2867void TargetDataOp::build(OpBuilder &builder, OperationState &state,
2868 const TargetDataOperands &clauses) {
2869 TargetDataOp::build(builder, state, clauses.device, clauses.ifExpr,
2870 clauses.mapVars, clauses.mapIterated,
2871 clauses.useDeviceAddrVars, clauses.useDevicePtrVars);
2872}
2873
2874LogicalResult TargetDataOp::verify() {
2875 if (getMapVars().empty() && getMapIterated().empty() &&
2876 getUseDevicePtrVars().empty() && getUseDeviceAddrVars().empty()) {
2877 return ::emitError(this->getLoc(),
2878 "At least one of map, use_device_ptr_vars, or "
2879 "use_device_addr_vars operand must be present");
2880 }
2881
2882 if (failed(verifyMapInfoDefinedArgs(*this, "use_device_ptr",
2883 getUseDevicePtrVars())))
2884 return failure();
2885
2886 if (failed(verifyMapInfoDefinedArgs(*this, "use_device_addr",
2887 getUseDeviceAddrVars())))
2888 return failure();
2889
2890 return verifyMapClause(*this, getMapVars(), getMapIterated());
2891}
2892
2893//===----------------------------------------------------------------------===//
2894// TargetEnterDataOp
2895//===----------------------------------------------------------------------===//
2896
2897void TargetEnterDataOp::build(
2898 OpBuilder &builder, OperationState &state,
2899 const TargetEnterExitUpdateDataOperands &clauses) {
2900 MLIRContext *ctx = builder.getContext();
2901 TargetEnterDataOp::build(
2902 builder, state, makeArrayAttr(ctx, clauses.dependKinds),
2903 clauses.dependVars, makeArrayAttr(ctx, clauses.dependIteratedKinds),
2904 clauses.dependIterated, clauses.device, clauses.ifExpr, clauses.mapVars,
2905 clauses.mapIterated, clauses.nowait);
2906}
2907
2908LogicalResult TargetEnterDataOp::verify() {
2909 LogicalResult verifyDependVars =
2910 verifyDependVarList(*this, getDependKinds(), getDependVars(),
2911 getDependIteratedKinds(), getDependIterated());
2912 return failed(verifyDependVars)
2913 ? verifyDependVars
2914 : verifyMapClause(*this, getMapVars(), getMapIterated());
2915}
2916
2917//===----------------------------------------------------------------------===//
2918// TargetExitDataOp
2919//===----------------------------------------------------------------------===//
2920
2921void TargetExitDataOp::build(OpBuilder &builder, OperationState &state,
2922 const TargetEnterExitUpdateDataOperands &clauses) {
2923 MLIRContext *ctx = builder.getContext();
2924 TargetExitDataOp::build(
2925 builder, state, makeArrayAttr(ctx, clauses.dependKinds),
2926 clauses.dependVars, makeArrayAttr(ctx, clauses.dependIteratedKinds),
2927 clauses.dependIterated, clauses.device, clauses.ifExpr, clauses.mapVars,
2928 clauses.mapIterated, clauses.nowait);
2929}
2930
2931LogicalResult TargetExitDataOp::verify() {
2932 LogicalResult verifyDependVars =
2933 verifyDependVarList(*this, getDependKinds(), getDependVars(),
2934 getDependIteratedKinds(), getDependIterated());
2935 return failed(verifyDependVars)
2936 ? verifyDependVars
2937 : verifyMapClause(*this, getMapVars(), getMapIterated());
2938}
2939
2940//===----------------------------------------------------------------------===//
2941// TargetUpdateOp
2942//===----------------------------------------------------------------------===//
2943
2944void TargetUpdateOp::build(OpBuilder &builder, OperationState &state,
2945 const TargetEnterExitUpdateDataOperands &clauses) {
2946 MLIRContext *ctx = builder.getContext();
2947 TargetUpdateOp::build(builder, state, makeArrayAttr(ctx, clauses.dependKinds),
2948 clauses.dependVars,
2949 makeArrayAttr(ctx, clauses.dependIteratedKinds),
2950 clauses.dependIterated, clauses.device, clauses.ifExpr,
2951 clauses.mapVars, clauses.mapIterated, clauses.nowait);
2952}
2953
2954LogicalResult TargetUpdateOp::verify() {
2955 LogicalResult verifyDependVars =
2956 verifyDependVarList(*this, getDependKinds(), getDependVars(),
2957 getDependIteratedKinds(), getDependIterated());
2958 return failed(verifyDependVars)
2959 ? verifyDependVars
2960 : verifyMapClause(*this, getMapVars(), getMapIterated());
2961}
2962
2963//===----------------------------------------------------------------------===//
2964// TargetOp
2965//===----------------------------------------------------------------------===//
2966
2967void TargetOp::build(OpBuilder &builder, OperationState &state,
2968 const TargetExtOperands &clauses) {
2969 MLIRContext *ctx = builder.getContext();
2970 TargetOp::build(
2971 builder, state, clauses.allocateVars, clauses.allocatorVars,
2972 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
2973 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
2974 makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,
2975 makeArrayAttr(ctx, clauses.dependIteratedKinds), clauses.dependIterated,
2976 clauses.device, clauses.dynGroupprivateAccessGroup,
2977 clauses.dynGroupprivateFallback, clauses.dynGroupprivateSize,
2978 clauses.hasDeviceAddrVars, clauses.hostEvalVars, clauses.ifExpr,
2979 clauses.inReductionVars,
2980 makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
2981 makeArrayAttr(ctx, clauses.inReductionSyms), clauses.isDevicePtrVars,
2982 clauses.mapVars, clauses.mapIterated, clauses.nowait, clauses.privateVars,
2983 makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
2984 clauses.threadLimitVars, /*private_maps=*/nullptr, clauses.kernelType);
2985}
2986
2987bool TargetOp::hasHostEvalTripCount() {
2988 TargetExecMode mode = getKernelType();
2989 if (mode == TargetExecMode::spmd || mode == TargetExecMode::spmd_no_loop)
2990 return true;
2991
2992 if (mode == TargetExecMode::bare)
2993 return false;
2994
2995 // If it represents a `target teams distribute` construct, also evaluate the
2996 // `distribute` trip count on the host.
2997 Operation *capturedOp =
2998 cast<ComposableOpInterface>(getOperation()).findCapturedOp();
2999 if (auto loopNestOp = dyn_cast_if_present<LoopNestOp>(capturedOp)) {
3001 loopNestOp.gatherWrappers(loopWrappers);
3002
3003 LoopWrapperInterface *innermostWrapper = loopWrappers.begin();
3004 if (isa<SimdOp>(innermostWrapper))
3005 innermostWrapper = std::next(innermostWrapper);
3006
3007 auto numWrappers = std::distance(innermostWrapper, loopWrappers.end());
3008 if (numWrappers != 1)
3009 return false;
3010
3011 if (!isa<DistributeOp>(innermostWrapper))
3012 return false;
3013
3014 Operation *parentOp = innermostWrapper->getOperation()->getParentOp();
3015 if (isa_and_present<TeamsOp>(parentOp) &&
3016 parentOp->getParentOp() == getOperation())
3017 return true;
3018 }
3019
3020 return false;
3021}
3022
3023/// An `omp.target` `in_reduction` operand is captured by a `map_entries` entry
3024/// when the entry's `MapInfoOp` var_ptr is the same SSA value, or another
3025/// result of the same defining op. At this stage, exact identity can only be
3026/// required for block arguments, which have no defining op. Flang emits
3027/// `hlfir.declare` #0 for the `in_reduction` operand and #1 for the map
3028/// `var_ptr`; these collapse to the same value after lowering, but that cannot
3029/// be enforced here.
3030static bool targetInReductionCapturedBy(Value inReductionVar, Value mapVarPtr) {
3031 if (mapVarPtr == inReductionVar)
3032 return true;
3033 Operation *def = inReductionVar.getDefiningOp();
3034 return def && mapVarPtr.getDefiningOp() == def;
3035}
3036
3037LogicalResult TargetOp::verify() {
3039 getOperation(), getAllocateVars(), getAllocatorVars(),
3040 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3041 getPrivateVars(), getPrivateSymsAttr())))
3042 return failure();
3043
3044 if (getKernelType() == TargetExecMode::bare && !isCombined())
3045 return emitOpError() << "bare kernel requires 'omp.combined'";
3046
3047 if (failed(verifyDependVarList(*this, getDependKinds(), getDependVars(),
3048 getDependIteratedKinds(),
3049 getDependIterated())))
3050 return failure();
3051
3052 if (failed(verifyMapInfoDefinedArgs(*this, "has_device_addr",
3053 getHasDeviceAddrVars())))
3054 return failure();
3055
3056 if (failed(verifyMapClause(*this, getMapVars(), getMapIterated())))
3057 return failure();
3058
3060 *this, getDynGroupprivateAccessGroupAttr(),
3061 getDynGroupprivateFallbackAttr(), getDynGroupprivateSize())))
3062 return failure();
3063
3064 if (failed(verifyPrivateVarList(*this)))
3065 return failure();
3066
3067 if (failed(verifyReductionVarList(*this, getInReductionSyms(),
3068 getInReductionVars(),
3069 getInReductionByref())))
3070 return failure();
3071
3072 // An `in_reduction` operand on `omp.target` has no dedicated entry block
3073 // argument; inside the region it is accessed through the block argument of a
3074 // matching `map_entries` entry, and the host rewrites that map argument to
3075 // the reduction-private storage. Require every `in_reduction` operand to be
3076 // captured by at least one `map_entries` entry.
3077 for (Value inReductionVar : getInReductionVars()) {
3078 bool captured = false;
3079 for (Value mapVar : getMapVars()) {
3080 auto mapInfo = mapVar.getDefiningOp<MapInfoOp>();
3081 if (targetInReductionCapturedBy(inReductionVar, mapInfo.getVarPtr())) {
3082 captured = true;
3083 break;
3084 }
3085 }
3086 if (!captured)
3087 return emitOpError() << "in_reduction variable must be captured by a "
3088 "matching map_entries entry";
3089 }
3090
3091 return verifyPrivateVarsMapping(*this);
3092}
3093
3094LogicalResult TargetOp::verifyRegions() {
3095 auto teamsOps = getOps<TeamsOp>();
3096 auto numNestedTeams = std::distance(teamsOps.begin(), teamsOps.end());
3097 if (numNestedTeams > 1)
3098 return emitError("target containing multiple 'omp.teams' nested ops");
3099
3100 if (numNestedTeams == 0) {
3101 switch (getKernelType()) {
3102 case TargetExecMode::bare:
3103 return emitOpError()
3104 << "bare kernel must contain a nested 'omp.teams' operation";
3105 case TargetExecMode::spmd_no_loop:
3106 return emitOpError() << "spmd_no_loop kernel must contain a nested "
3107 "'omp.teams' operation";
3108 default:
3109 break;
3110 }
3111 }
3112
3113 Operation *capturedOp =
3114 cast<ComposableOpInterface>(getOperation()).findCapturedOp();
3115 if ((getKernelType() == TargetExecMode::spmd ||
3116 getKernelType() == TargetExecMode::spmd_no_loop) &&
3117 !isa_and_present<LoopNestOp>(capturedOp))
3118 return emitOpError()
3119 << "SPMD kernel must capture an 'omp.loop_nest' operation";
3120
3121 bool isTargetDevice = false;
3122 if (auto offloadMod = (*this)->getParentOfType<OffloadModuleInterface>())
3123 if (offloadMod.getIsTargetDevice())
3124 isTargetDevice = true;
3125
3126 // Check that host_eval values are only used in legal ways.
3127 llvm::ArrayRef<BlockArgument> hostEvalBlockArgs =
3128 cast<BlockArgOpenMPOpInterface>(getOperation()).getHostEvalBlockArgs();
3129
3130 bool hostEvalTripCount = hasHostEvalTripCount();
3131 for (Value hostEvalArg : hostEvalBlockArgs) {
3132 for (Operation *user : hostEvalArg.getUsers()) {
3133 if (auto teamsOp = dyn_cast<TeamsOp>(user)) {
3134 // Check if used in num_teams_lower or any of num_teams_upper_vars
3135 if (hostEvalArg == teamsOp.getNumTeamsLower() ||
3136 llvm::is_contained(teamsOp.getNumTeamsUpperVars(), hostEvalArg) ||
3137 llvm::is_contained(teamsOp.getThreadLimitVars(), hostEvalArg))
3138 continue;
3139
3140 return emitOpError() << "host_eval argument only legal as 'num_teams' "
3141 "and 'thread_limit' in 'omp.teams'";
3142 }
3143 if (auto parallelOp = dyn_cast<ParallelOp>(user)) {
3144 if (llvm::is_contained(parallelOp.getNumThreadsVars(), hostEvalArg))
3145 continue;
3146
3147 return emitOpError()
3148 << "host_eval argument only legal as 'num_threads' in "
3149 "'omp.parallel'";
3150 }
3151 if (auto loopNestOp = dyn_cast<LoopNestOp>(user)) {
3152 if (hostEvalTripCount &&
3153 (llvm::is_contained(loopNestOp.getLoopLowerBounds(), hostEvalArg) ||
3154 llvm::is_contained(loopNestOp.getLoopUpperBounds(), hostEvalArg) ||
3155 llvm::is_contained(loopNestOp.getLoopSteps(), hostEvalArg)))
3156 continue;
3157
3158 return emitOpError() << "host_eval argument only legal as loop bounds "
3159 "and steps in 'omp.loop_nest' when trip count "
3160 "must be evaluated in the host";
3161 }
3162
3163 return emitOpError() << "host_eval argument illegal use in '"
3164 << user->getName() << "' operation";
3165 }
3166 }
3167
3168 if (hostEvalTripCount && !isTargetDevice) {
3169 auto loopOp = cast<LoopNestOp>(capturedOp);
3170 for (auto arg : llvm::concat<Value>(loopOp.getLoopLowerBounds(),
3171 loopOp.getLoopUpperBounds(),
3172 loopOp.getLoopSteps())) {
3173 if (!llvm::is_contained(hostEvalBlockArgs, arg))
3174 return emitOpError() << "nested 'omp.loop_nest' bounds expected to "
3175 "be host-evaluated";
3176 }
3177 }
3178
3179 return success();
3180}
3181
3182//===----------------------------------------------------------------------===//
3183// ParallelOp
3184//===----------------------------------------------------------------------===//
3185
3186void ParallelOp::build(OpBuilder &builder, OperationState &state,
3187 ArrayRef<NamedAttribute> attributes) {
3188 ParallelOp::build(builder, state, /*allocate_vars=*/ValueRange(),
3189 /*allocator_vars=*/ValueRange(),
3190 /*allocate_alignments=*/nullptr,
3191 /*allocate_private_indices=*/nullptr, /*if_expr=*/nullptr,
3192 /*num_threads_vars=*/ValueRange(),
3193 /*private_vars=*/ValueRange(),
3194 /*private_syms=*/nullptr, /*private_needs_barrier=*/false,
3195 /*proc_bind_kind=*/nullptr,
3196 /*reduction_mods =*/nullptr,
3197 /*reduction_vars=*/ValueRange(),
3198 /*reduction_byref=*/nullptr, /*reduction_syms=*/nullptr);
3199 state.addAttributes(attributes);
3200}
3201
3202void ParallelOp::build(OpBuilder &builder, OperationState &state,
3203 const ParallelOperands &clauses) {
3204 MLIRContext *ctx = builder.getContext();
3205 ParallelOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
3206 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
3207 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
3208 clauses.ifExpr, clauses.numThreadsVars, clauses.privateVars,
3209 makeArrayAttr(ctx, clauses.privateSyms),
3210 clauses.privateNeedsBarrier, clauses.procBindKind,
3211 makeArrayAttr(ctx, clauses.reductionMods),
3212 clauses.reductionVars,
3213 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
3214 makeArrayAttr(ctx, clauses.reductionSyms));
3215}
3216
3217template <typename OpType>
3218static LogicalResult verifyPrivateVarList(OpType &op) {
3219 auto privateVars = op.getPrivateVars();
3220 auto privateSyms = op.getPrivateSymsAttr();
3221
3222 if (privateVars.empty() && (privateSyms == nullptr || privateSyms.empty()))
3223 return success();
3224
3225 auto numPrivateVars = privateVars.size();
3226 auto numPrivateSyms = (privateSyms == nullptr) ? 0 : privateSyms.size();
3227
3228 if (numPrivateVars != numPrivateSyms)
3229 return op.emitError() << "inconsistent number of private variables and "
3230 "privatizer op symbols, private vars: "
3231 << numPrivateVars
3232 << " vs. privatizer op symbols: " << numPrivateSyms;
3233
3234 for (auto privateVarInfo : llvm::zip_equal(privateVars, privateSyms)) {
3235 Type varType = std::get<0>(privateVarInfo).getType();
3236 SymbolRefAttr privateSym = cast<SymbolRefAttr>(std::get<1>(privateVarInfo));
3237 PrivateClauseOp privatizerOp =
3239
3240 if (privatizerOp == nullptr)
3241 return op.emitError() << "failed to lookup privatizer op with symbol: '"
3242 << privateSym << "'";
3243
3244 Type privatizerType = privatizerOp.getArgType();
3245
3246 if (privatizerType && (varType != privatizerType))
3247 return op.emitError()
3248 << "type mismatch between a "
3249 << stringifyDataSharingClauseType(
3250 privatizerOp.getDataSharingType())
3251 << " variable and its privatizer op, var type: " << varType
3252 << " vs. privatizer op type: " << privatizerType;
3253
3254 if (privatizerOp.getDataSharingType() == DataSharingClauseType::Linear)
3255 return op.emitError() << "`linear` privatizer '" << privateSym
3256 << "' can only be used in a `linear` clause";
3257 }
3258
3259 return success();
3260}
3261
3262/// Verifies that each linear variable has a `linear` privatizer.
3263template <typename OpType>
3264static LogicalResult verifyLinearSyms(OpType op) {
3265 OperandRange linearVars = op.getLinearVars();
3266 std::optional<ArrayAttr> linearSyms = op.getLinearSyms();
3267 if (linearVars.empty() && (!linearSyms || linearSyms->empty()))
3268 return success();
3269
3270 if (!linearSyms || linearSyms->size() != linearVars.size())
3271 return op.emitOpError()
3272 << "expected as many linear privatizers as linear variables";
3273
3274 for (auto [var, sym] : llvm::zip_equal(linearVars, *linearSyms)) {
3275 auto symRef = cast<SymbolRefAttr>(sym);
3276 auto privatizer =
3278 if (!privatizer)
3279 return op.emitOpError() << "failed to lookup privatizer op with symbol: '"
3280 << symRef << "'";
3281 if (privatizer.getDataSharingType() != DataSharingClauseType::Linear)
3282 return op.emitOpError() << "privatizer '" << symRef
3283 << "' of a linear variable must be `linear`";
3284 Type argType = privatizer.getArgType();
3285 if (argType && argType != var.getType())
3286 return op.emitOpError()
3287 << "type mismatch between a linear variable and its privatizer "
3288 "op, var type: "
3289 << var.getType() << " vs. privatizer op type: " << argType;
3290 }
3291 return success();
3292}
3293
3294LogicalResult ParallelOp::verify() {
3295 if (failed(verifyPrivateVarList(*this)))
3296 return failure();
3298 getOperation(), getAllocateVars(), getAllocatorVars(),
3299 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3300 getPrivateVars(), getPrivateSymsAttr(),
3301 /*requirePrivateIndices=*/true)))
3302 return failure();
3303
3304 return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
3305 getReductionByref(), getReductionModsAttr());
3306}
3307
3308LogicalResult ParallelOp::verifyRegions() {
3309 auto distChildOps = getOps<DistributeOp>();
3310 int numDistChildOps = std::distance(distChildOps.begin(), distChildOps.end());
3311 if (numDistChildOps > 1)
3312 return emitError()
3313 << "multiple 'omp.distribute' nested inside of 'omp.parallel'";
3314
3315 if (numDistChildOps == 1) {
3316 if (!isComposite())
3317 return emitError()
3318 << "'omp.composite' attribute missing from composite operation";
3319
3320 auto *ompDialect = getContext()->getLoadedDialect<OpenMPDialect>();
3321 Operation &distributeOp = **distChildOps.begin();
3322 for (Operation &childOp : getOps()) {
3323 if (&childOp == &distributeOp || ompDialect != childOp.getDialect())
3324 continue;
3325
3326 if (!childOp.hasTrait<OpTrait::IsTerminator>())
3327 return emitError() << "unexpected OpenMP operation inside of composite "
3328 "'omp.parallel': "
3329 << childOp.getName();
3330 }
3331 } else if (isComposite()) {
3332 return emitError()
3333 << "'omp.composite' attribute present in non-composite operation";
3334 }
3335 return success();
3336}
3337
3338//===----------------------------------------------------------------------===//
3339// TeamsOp
3340//===----------------------------------------------------------------------===//
3341
3343 while ((op = op->getParentOp()))
3344 if (isa<OpenMPDialect>(op->getDialect()))
3345 return false;
3346 return true;
3347}
3348
3349void TeamsOp::build(OpBuilder &builder, OperationState &state,
3350 const TeamsOperands &clauses) {
3351 MLIRContext *ctx = builder.getContext();
3352 // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
3353 TeamsOp::build(
3354 builder, state, clauses.allocateVars, clauses.allocatorVars,
3355 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
3356 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
3357 clauses.dynGroupprivateAccessGroup, clauses.dynGroupprivateFallback,
3358 clauses.dynGroupprivateSize, clauses.ifExpr, clauses.numTeamsLower,
3359 clauses.numTeamsUpperVars, /*private_vars=*/{}, /*private_syms=*/nullptr,
3360 /*private_needs_barrier=*/false,
3361 makeArrayAttr(ctx, clauses.reductionMods), clauses.reductionVars,
3362 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
3363 makeArrayAttr(ctx, clauses.reductionSyms), clauses.threadLimitVars);
3364}
3365
3366// Verify num_teams clause
3367static LogicalResult verifyNumTeamsClause(Operation *op, Value numTeamsLower,
3368 OperandRange numTeamsUpperVars) {
3369 // If lower is specified, upper must have exactly one value
3370 if (numTeamsLower) {
3371 if (numTeamsUpperVars.size() != 1)
3372 return op->emitError(
3373 "expected exactly one num_teams upper bound when lower bound is "
3374 "specified");
3375 if (numTeamsLower.getType() != numTeamsUpperVars[0].getType())
3376 return op->emitError(
3377 "expected num_teams upper bound and lower bound to be "
3378 "the same type");
3379 }
3380
3381 return success();
3382}
3383
3384LogicalResult TeamsOp::verify() {
3385 // Check parent region
3386 // TODO If nested inside of a target region, also check that it does not
3387 // contain any statements, declarations or directives other than this
3388 // omp.teams construct. The issue is how to support the initialization of
3389 // this operation's own arguments (allow SSA values across omp.target?).
3390 Operation *op = getOperation();
3391 auto parentTarget = llvm::dyn_cast_if_present<TargetOp>(op->getParentOp());
3392 if (!parentTarget && !opInGlobalImplicitParallelRegion(op))
3393 return emitError("expected to be nested inside of omp.target or not nested "
3394 "in any OpenMP dialect operations");
3395
3396 // Check for num_teams clause restrictions
3397 if (failed(verifyNumTeamsClause(op, this->getNumTeamsLower(),
3398 this->getNumTeamsUpperVars())))
3399 return failure();
3400
3401 if (parentTarget &&
3402 parentTarget.getKernelType() == TargetExecMode::spmd_no_loop &&
3403 (getNumTeamsLower() || !getNumTeamsUpperVars().empty()))
3404 return emitOpError() << "'num_teams' not allowed in SPMD-no-loop kernels";
3405
3407 getOperation(), getAllocateVars(), getAllocatorVars(),
3408 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3409 getPrivateVars(), getPrivateSymsAttr())))
3410 return failure();
3411
3413 op, getDynGroupprivateAccessGroupAttr(),
3414 getDynGroupprivateFallbackAttr(), getDynGroupprivateSize())))
3415 return failure();
3416
3417 if (failed(verifyPrivateVarList(*this)))
3418 return failure();
3419
3420 return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
3421 getReductionByref(), getReductionModsAttr());
3422}
3423
3424//===----------------------------------------------------------------------===//
3425// SectionOp
3426//===----------------------------------------------------------------------===//
3427
3428OperandRange SectionOp::getPrivateVars() {
3429 return getParentOp().getPrivateVars();
3430}
3431
3432OperandRange SectionOp::getReductionVars() {
3433 return getParentOp().getReductionVars();
3434}
3435
3436//===----------------------------------------------------------------------===//
3437// SectionsOp
3438//===----------------------------------------------------------------------===//
3439
3440void SectionsOp::build(OpBuilder &builder, OperationState &state,
3441 const SectionsOperands &clauses) {
3442 MLIRContext *ctx = builder.getContext();
3443 // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
3444 SectionsOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
3445 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
3446 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
3447 clauses.nowait, /*private_vars=*/{},
3448 /*private_syms=*/nullptr, /*private_needs_barrier=*/nullptr,
3449 /*private_assume_bounded_recipe_effects=*/nullptr,
3450 makeArrayAttr(ctx, clauses.reductionMods),
3451 clauses.reductionVars,
3452 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
3453 makeArrayAttr(ctx, clauses.reductionSyms),
3454 /*reduction_assume_bounded_recipe_effects=*/nullptr);
3455}
3456
3457LogicalResult SectionsOp::verify() {
3458 if (isCombined())
3459 return emitOpError() << "cannot be a non-innermost combined construct leaf";
3460
3462 getOperation(), getAllocateVars(), getAllocatorVars(),
3463 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3464 getPrivateVars(), getPrivateSymsAttr())))
3465 return failure();
3466
3467 return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
3468 getReductionByref(), getReductionModsAttr());
3469}
3470
3471LogicalResult SectionsOp::verifyRegions() {
3472 for (auto &inst : *getRegion().begin()) {
3473 if (!(isa<SectionOp>(inst) || isa<TerminatorOp>(inst))) {
3474 return emitOpError()
3475 << "expected omp.section op or terminator op inside region";
3476 }
3477 }
3478
3479 return success();
3480}
3481
3482/// Add arbitrary read, write, allocate, and free effects.
3485 effects.emplace_back(MemoryEffects::Read::get());
3486 effects.emplace_back(MemoryEffects::Write::get());
3487 effects.emplace_back(MemoryEffects::Allocate::get());
3488 effects.emplace_back(MemoryEffects::Free::get());
3489}
3490
3491/// Privatisation may read the original as a mold or firstprivate copy source.
3492/// Any lastprivate copy-back is explicit in the nested operations.
3495 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(op);
3496
3497 OperandRange sourceVars = iface.getPrivateVars();
3498 for (OpOperand &source :
3499 MutableArrayRef(sourceVars.getBase(), sourceVars.size()))
3500 effects.emplace_back(MemoryEffects::Read::get(), &source);
3501 for (BlockArgument priv : iface.getPrivateBlockArgs()) {
3502 effects.emplace_back(MemoryEffects::Write::get(), priv);
3503 effects.emplace_back(MemoryEffects::Allocate::get(), priv);
3504 effects.emplace_back(MemoryEffects::Free::get(), priv);
3505 }
3506}
3507
3508/// Reductions initialise private accumulators and combine them into the
3509/// original variables. Account for reduction and in_reduction clauses.
3510static void
3513 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(op);
3514
3515 auto addEffects = [&](OperandRange sourceVars,
3516 ArrayRef<BlockArgument> privateArgs) {
3517 for (OpOperand &source :
3518 MutableArrayRef(sourceVars.getBase(), sourceVars.size())) {
3519 effects.emplace_back(MemoryEffects::Read::get(), &source);
3520 effects.emplace_back(MemoryEffects::Write::get(), &source);
3521 }
3522
3523 for (BlockArgument priv : privateArgs) {
3524 effects.emplace_back(MemoryEffects::Allocate::get(), priv);
3525 effects.emplace_back(MemoryEffects::Free::get(), priv);
3526 effects.emplace_back(MemoryEffects::Read::get(), priv);
3527 effects.emplace_back(MemoryEffects::Write::get(), priv);
3528 }
3529 };
3530
3531 addEffects(iface.getReductionVars(), iface.getReductionBlockArgs());
3532 addEffects(iface.getInReductionVars(), iface.getInReductionBlockArgs());
3533}
3534
3537 &effects) {
3538 effects.emplace_back(MemoryEffects::Read::get());
3539 effects.emplace_back(MemoryEffects::Write::get());
3540}
3541
3542void SectionsOp::getEffects(
3544 &effects) {
3545 // The presence of the nowait clause eliminates the implicit barrier at the
3546 // end of a sections construct
3547 if (!getNowait()) {
3548 getBarrierEffects(effects);
3549 }
3550
3551 bool hasUnknownRecipeEffects =
3552 (!getPrivateVars().empty() && !getPrivateAssumeBoundedRecipeEffects()) ||
3553 (!getReductionVars().empty() &&
3554 !getReductionAssumeBoundedRecipeEffects());
3555
3556 if (hasUnknownRecipeEffects) {
3557 addUnknownMemoryEffects(effects);
3558 } else {
3559 getPrivatisationEffects(*this, effects);
3560 getReductionEffects(*this, effects);
3561 }
3562}
3563
3564//===----------------------------------------------------------------------===//
3565// ScopeOp
3566//===----------------------------------------------------------------------===//
3567
3568void ScopeOp::build(OpBuilder &builder, OperationState &state,
3569 const ScopeOperands &clauses) {
3570 MLIRContext *ctx = builder.getContext();
3571 ScopeOp::build(
3572 builder, state, clauses.allocateVars, clauses.allocatorVars,
3573 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
3574 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
3575 clauses.nowait, clauses.privateVars,
3576 makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
3577 makeArrayAttr(ctx, clauses.reductionMods), clauses.reductionVars,
3578 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
3579 makeArrayAttr(ctx, clauses.reductionSyms));
3580}
3581
3582LogicalResult ScopeOp::verify() {
3584 getOperation(), getAllocateVars(), getAllocatorVars(),
3585 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3586 getPrivateVars(), getPrivateSymsAttr(),
3587 /*requirePrivateIndices=*/true)))
3588 return failure();
3589
3590 if (failed(verifyPrivateVarList(*this)))
3591 return failure();
3592
3593 return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
3594 getReductionByref(), getReductionModsAttr());
3595}
3596
3597//===----------------------------------------------------------------------===//
3598// SingleOp
3599//===----------------------------------------------------------------------===//
3600
3601void SingleOp::build(OpBuilder &builder, OperationState &state,
3602 const SingleOperands &clauses) {
3603 MLIRContext *ctx = builder.getContext();
3604 // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
3605 SingleOp::build(
3606 builder, state, clauses.allocateVars, clauses.allocatorVars,
3607 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
3608 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
3609 clauses.copyprivateVars, makeArrayAttr(ctx, clauses.copyprivateSyms),
3610 /*copyprivate_assume_bounded_recipe_effects=*/nullptr, clauses.nowait,
3611 /*private_vars=*/{}, /*private_syms=*/nullptr,
3612 /*private_needs_barrier=*/nullptr,
3613 /*private_assume_bounded_recipe_effects=*/nullptr);
3614}
3615
3616LogicalResult SingleOp::verify() {
3618 getOperation(), getAllocateVars(), getAllocatorVars(),
3619 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3620 getPrivateVars(), getPrivateSymsAttr())))
3621 return failure();
3622
3623 return verifyCopyprivateVarList(*this, getCopyprivateVars(),
3624 getCopyprivateSyms());
3625}
3626
3627void SingleOp::getEffects(
3629 &effects) {
3630 // The presence of the nowait clause eliminates the implicit barrier at the
3631 // end of a single construct.
3632 if (!getNowait()) {
3633 getBarrierEffects(effects);
3634 }
3635
3636 bool hasUnknownRecipeEffects =
3637 (!getPrivateVars().empty() && !getPrivateAssumeBoundedRecipeEffects()) ||
3638 (!getCopyprivateVars().empty() &&
3639 !getCopyprivateAssumeBoundedRecipeEffects());
3640 if (hasUnknownRecipeEffects) {
3641 addUnknownMemoryEffects(effects);
3642 } else {
3643 getPrivatisationEffects(*this, effects);
3644 // The unknown executing thread supplies the value and the other threads
3645 // receive it, so each copyprivate operand may be both read and written.
3646 OperandRange copyprivateVars = getCopyprivateVars();
3647 for (OpOperand &variable :
3648 MutableArrayRef(copyprivateVars.getBase(), copyprivateVars.size())) {
3649 effects.emplace_back(MemoryEffects::Read::get(), &variable);
3650 effects.emplace_back(MemoryEffects::Write::get(), &variable);
3651 }
3652 }
3653}
3654
3655//===----------------------------------------------------------------------===//
3656// WorkshareOp
3657//===----------------------------------------------------------------------===//
3658
3659void WorkshareOp::build(OpBuilder &builder, OperationState &state,
3660 const WorkshareOperands &clauses) {
3661 WorkshareOp::build(builder, state, clauses.nowait);
3662}
3663
3664LogicalResult WorkshareOp::verify() {
3665 if (isCombined())
3666 return emitOpError() << "cannot be a non-innermost combined construct leaf";
3667
3668 return success();
3669}
3670
3671//===----------------------------------------------------------------------===//
3672// WorkshareLoopWrapperOp
3673//===----------------------------------------------------------------------===//
3674
3675LogicalResult WorkshareLoopWrapperOp::verifyRegions() {
3676 if (isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||
3677 getNestedWrapper())
3678 return emitOpError() << "expected to be a standalone loop wrapper";
3679
3680 return success();
3681}
3682
3683//===----------------------------------------------------------------------===//
3684// LoopWrapperInterface
3685//===----------------------------------------------------------------------===//
3686
3687LogicalResult LoopWrapperInterface::verifyImpl() {
3688 Operation *op = this->getOperation();
3689 if (!op->hasTrait<OpTrait::NoTerminator>() ||
3691 return emitOpError() << "loop wrapper must also have the `NoTerminator` "
3692 "and `SingleBlock` traits";
3693
3694 if (op->getNumRegions() != 1)
3695 return emitOpError() << "loop wrapper does not contain exactly one region";
3696
3697 Region &region = op->getRegion(0);
3698 if (range_size(region.getOps()) != 1)
3699 return emitOpError()
3700 << "loop wrapper does not contain exactly one nested op";
3701
3702 Operation &firstOp = *region.op_begin();
3703 if (!isa<LoopNestOp, LoopWrapperInterface>(firstOp))
3704 return emitOpError() << "nested in loop wrapper is not another loop "
3705 "wrapper or `omp.loop_nest`";
3706
3707 return success();
3708}
3709
3710//===----------------------------------------------------------------------===//
3711// ComposableOpInterface
3712//===----------------------------------------------------------------------===//
3713
3714Operation *ComposableOpInterface::findCapturedOp() {
3715 Operation *op = this->getOperation();
3716
3717 // Handle the composite case by returning the wrapped omp.loop_nest.
3718 if (auto wrapperOp = dyn_cast<LoopWrapperInterface>(op))
3719 return wrapperOp.getWrappedLoop();
3720
3721 // Do not look further if this op is not combined with any of its children.
3722 // Need to check for composite for the omp.parallel case, which is not a loop
3723 // wrapper itself.
3724 if (!isCombined() && !isComposite())
3725 return op;
3726
3727 Region &region = op->getRegion(0);
3728 for (Operation &nestedOp : region.getOps()) {
3729 if (auto wrapperOp = dyn_cast<LoopWrapperInterface>(&nestedOp))
3730 return wrapperOp.getWrappedLoop();
3731
3732 if (auto composableOp = dyn_cast<ComposableOpInterface>(&nestedOp))
3733 return composableOp.findCapturedOp();
3734 }
3735
3736 // This can only be reached if the op has an omp.combined attribute but the
3737 // corresponding nested composable op has been deleted. In that case, it's
3738 // correct to return this operation.
3739 return op;
3740}
3741
3742LogicalResult ComposableOpInterface::verifyImpl() {
3743 Operation *op = this->getOperation();
3744
3745 if (op->getNumRegions() != 1)
3746 return emitOpError() << "composable ops must have a single region";
3747
3748 if (isComposite() && !isa<LoopWrapperInterface, ParallelOp>(op))
3749 return emitOpError() << "non-loop wrapper cannot be composite";
3750
3751 // If combined, must have exactly one eligible nested op (composable or loop
3752 // wrapper).
3753 if (isCombined()) {
3754 Operation *nestedOp = nullptr;
3755 auto count = llvm::count_if(
3756 op->getRegion(0).getOps(), [&nestedOp](mlir::Operation &op) {
3757 if (isa<ComposableOpInterface, LoopWrapperInterface>(op)) {
3758 nestedOp = &op;
3759 return true;
3760 }
3761 return false;
3762 });
3763
3764 // Make an exception for ops marked as omp.combined with no eligible nested
3765 // ops: this situation should be disallowed, but it can be reached if an
3766 // MLIR optimization pass find that the child operation has no side effects
3767 // (many ComposableOpInterface ops have RecursiveMemoryEffects), so it gets
3768 // deleted without updating the parent's attribute.
3769 //
3770 // Since there's a well defined way of handling that situation (treat it as
3771 // non-combined), we relax the requirement here. Ensuring the parent is
3772 // updated every time a pass that can potentially remove a child composable
3773 // op runs is less preferable as a solution.
3774 if (count == 0)
3775 return success();
3776
3777 if (count > 1)
3778 return emitOpError()
3779 << "multiple eligible child ops found in combined op";
3780
3781 // This operation cannot be combined if its captured nested op can be
3782 // executed more than once (i.e. its block's successors can reach it) or if
3783 // it's not guaranteed to be executed before all exits of the region (i.e.
3784 // it doesn't dominate all blocks with no successors reachable from the
3785 // entry block).
3786 DominanceInfo domInfo;
3787 Block *parentBlock = nestedOp->getBlock();
3788
3789 for (Block *successor : parentBlock->getSuccessors())
3790 if (successor->isReachable(parentBlock))
3791 return emitOpError() << "nested combined child op is part of a loop";
3792
3793 for (Block &block : op->getRegion(0))
3794 if (domInfo.isReachableFromEntry(&block) && block.hasNoSuccessors() &&
3795 !domInfo.dominates(parentBlock, &block))
3796 return emitOpError()
3797 << "nested combined child op doesn't unconditionally execute";
3798 }
3799 return success();
3800}
3801
3802//===----------------------------------------------------------------------===//
3803// LoopOp
3804//===----------------------------------------------------------------------===//
3805
3806void LoopOp::build(OpBuilder &builder, OperationState &state,
3807 const LoopOperands &clauses) {
3808 MLIRContext *ctx = builder.getContext();
3809
3810 LoopOp::build(builder, state, clauses.bindKind, clauses.privateVars,
3811 makeArrayAttr(ctx, clauses.privateSyms),
3812 clauses.privateNeedsBarrier, clauses.order, clauses.orderMod,
3813 makeArrayAttr(ctx, clauses.reductionMods),
3814 clauses.reductionVars,
3815 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
3816 makeArrayAttr(ctx, clauses.reductionSyms));
3817}
3818
3819LogicalResult LoopOp::verify() {
3820 if (failed(verifyPrivateVarList(*this)))
3821 return failure();
3822
3823 return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
3824 getReductionByref(), getReductionModsAttr());
3825}
3826
3827LogicalResult LoopOp::verifyRegions() {
3828 if (llvm::isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||
3829 getNestedWrapper())
3830 return emitOpError() << "expected to be a standalone loop wrapper";
3831
3832 return success();
3833}
3834
3835//===----------------------------------------------------------------------===//
3836// WsloopOp
3837//===----------------------------------------------------------------------===//
3838
3839void WsloopOp::build(OpBuilder &builder, OperationState &state,
3840 ArrayRef<NamedAttribute> attributes) {
3841 build(builder, state, /*allocate_vars=*/{}, /*allocator_vars=*/{},
3842 /*allocate_alignments=*/nullptr,
3843 /*allocate_private_indices=*/nullptr,
3844 /*linear_vars=*/ValueRange(), /*linear_step_vars=*/ValueRange(),
3845 /*linear_var_types*/ nullptr, /*linear_modifiers=*/nullptr,
3846 /*linear_syms=*/nullptr, /*nowait=*/false, /*order=*/nullptr,
3847 /*order_mod=*/nullptr,
3848 /*ordered=*/nullptr, /*private_vars=*/{}, /*private_syms=*/nullptr,
3849 /*private_needs_barrier=*/false,
3850 /*reduction_mods=*/nullptr, /*reduction_vars=*/ValueRange(),
3851 /*reduction_byref=*/nullptr,
3852 /*reduction_syms=*/nullptr, /*schedule_kind=*/nullptr,
3853 /*schedule_chunk=*/nullptr, /*schedule_mod=*/nullptr,
3854 /*schedule_simd=*/false);
3855 state.addAttributes(attributes);
3856}
3857
3858void WsloopOp::build(OpBuilder &builder, OperationState &state,
3859 const WsloopOperands &clauses) {
3860 MLIRContext *ctx = builder.getContext();
3861 WsloopOp::build(
3862 builder, state, clauses.allocateVars, clauses.allocatorVars,
3863 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
3864 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
3865 clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
3866 clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
3867 clauses.nowait, clauses.order, clauses.orderMod, clauses.ordered,
3868 clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
3869 clauses.privateNeedsBarrier, makeArrayAttr(ctx, clauses.reductionMods),
3870 clauses.reductionVars,
3871 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
3872 makeArrayAttr(ctx, clauses.reductionSyms), clauses.scheduleKind,
3873 clauses.scheduleChunk, clauses.scheduleMod, clauses.scheduleSimd);
3874}
3875
3876LogicalResult WsloopOp::verify() {
3878 getOperation(), getAllocateVars(), getAllocatorVars(),
3879 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
3880 getPrivateVars(), getPrivateSymsAttr())))
3881 return failure();
3882
3883 if (failed(
3884 verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars())))
3885 return failure();
3886 if (getLinearVars().size() &&
3887 getLinearVarTypes().value().size() != getLinearVars().size())
3888 return emitError() << "Ill-formed type attributes for linear variables";
3889
3890 if (failed(verifyLinearSyms(*this)) || failed(verifyPrivateVarList(*this)))
3891 return failure();
3892
3893 return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
3894 getReductionByref(), getReductionModsAttr());
3895}
3896
3897LogicalResult WsloopOp::verifyRegions() {
3898 bool isCompositeChildLeaf =
3899 llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
3900
3901 if (LoopWrapperInterface nested = getNestedWrapper()) {
3902 if (!isComposite())
3903 return emitError()
3904 << "'omp.composite' attribute missing from composite wrapper";
3905
3906 // Check for the allowed leaf constructs that may appear in a composite
3907 // construct directly after DO/FOR.
3908 if (!isa<SimdOp>(nested))
3909 return emitError() << "only supported nested wrapper is 'omp.simd'";
3910
3911 } else if (isComposite() && !isCompositeChildLeaf) {
3912 return emitError()
3913 << "'omp.composite' attribute present in non-composite wrapper";
3914 } else if (!isComposite() && isCompositeChildLeaf) {
3915 return emitError()
3916 << "'omp.composite' attribute missing from composite wrapper";
3917 }
3918
3919 return success();
3920}
3921
3922//===----------------------------------------------------------------------===//
3923// Simd construct [2.9.3.1]
3924//===----------------------------------------------------------------------===//
3925
3926void SimdOp::build(OpBuilder &builder, OperationState &state,
3927 const SimdOperands &clauses) {
3928 MLIRContext *ctx = builder.getContext();
3929 SimdOp::build(
3930 builder, state, clauses.alignedVars,
3931 makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,
3932 clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
3933 clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
3934 clauses.nontemporalVars, clauses.order, clauses.orderMod,
3935 clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
3936 clauses.privateNeedsBarrier, makeArrayAttr(ctx, clauses.reductionMods),
3937 clauses.reductionVars,
3938 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
3939 makeArrayAttr(ctx, clauses.reductionSyms), clauses.safelen,
3940 clauses.simdlen);
3941}
3942
3943LogicalResult SimdOp::verify() {
3944 if (failed(verifyReductionModifiers(*this, getReductionVars(),
3945 getReductionModsAttr())))
3946 return failure();
3947
3948 if (getSimdlen().has_value() && getSafelen().has_value() &&
3949 getSimdlen().value() > getSafelen().value())
3950 return emitOpError()
3951 << "simdlen clause and safelen clause are both present, but the "
3952 "simdlen value is not less than or equal to safelen value";
3953
3954 if (verifyAlignedClause(*this, getAlignments(), getAlignedVars()).failed())
3955 return failure();
3956
3957 if (verifyNontemporalClause(*this, getNontemporalVars()).failed())
3958 return failure();
3959
3960 if (failed(
3961 verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars())))
3962 return failure();
3963
3964 bool isCompositeChildLeaf =
3965 llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
3966
3967 if (!isComposite() && isCompositeChildLeaf)
3968 return emitError()
3969 << "'omp.composite' attribute missing from composite wrapper";
3970
3971 if (isComposite() && !isCompositeChildLeaf)
3972 return emitError()
3973 << "'omp.composite' attribute present in non-composite wrapper";
3974
3975 // Firstprivate is not allowed for SIMD in the standard. Check that none of
3976 // the private decls are for firstprivate.
3977 std::optional<ArrayAttr> privateSyms = getPrivateSyms();
3978 if (privateSyms) {
3979 for (const Attribute &sym : *privateSyms) {
3980 auto symRef = cast<SymbolRefAttr>(sym);
3981 omp::PrivateClauseOp privatizer =
3983 getOperation(), symRef);
3984 if (!privatizer)
3985 return emitError() << "Cannot find privatizer '" << symRef << "'";
3986 if (privatizer.getDataSharingType() ==
3987 DataSharingClauseType::FirstPrivate)
3988 return emitError() << "FIRSTPRIVATE cannot be used with SIMD";
3989 }
3990 }
3991
3992 if (failed(verifyPrivateVarList(*this)))
3993 return failure();
3994
3995 if (getLinearVars().size() &&
3996 getLinearVarTypes().value().size() != getLinearVars().size())
3997 return emitError() << "Ill-formed type attributes for linear variables";
3998
3999 if (failed(verifyLinearSyms(*this)))
4000 return failure();
4001
4002 llvm::DenseSet<Value> privateVars(llvm::from_range, getPrivateVars());
4003 llvm::DenseSet<Value> reductionVars(llvm::from_range, getReductionVars());
4004 // TODO Check lastprivate vars when their support is added to SimdOp.
4005 for (Value var : getLinearVars()) {
4006 if (privateVars.contains(var) || reductionVars.contains(var))
4007 return emitOpError()
4008 << "linear variables cannot appear in other data-sharing clauses";
4009 }
4010
4011 return success();
4012}
4013
4014LogicalResult SimdOp::verifyRegions() {
4015 if (getNestedWrapper())
4016 return emitOpError() << "must wrap an 'omp.loop_nest' directly";
4017
4018 return success();
4019}
4020
4021//===----------------------------------------------------------------------===//
4022// Distribute construct [2.9.4.1]
4023//===----------------------------------------------------------------------===//
4024
4025void DistributeOp::build(OpBuilder &builder, OperationState &state,
4026 const DistributeOperands &clauses) {
4027 DistributeOp::build(
4028 builder, state, clauses.allocateVars, clauses.allocatorVars,
4029 makeDenseI64ArrayAttr(builder.getContext(), clauses.allocateAlignments),
4031 clauses.allocatePrivateIndices),
4032 clauses.distScheduleStatic, clauses.distScheduleChunkSize, clauses.order,
4033 clauses.orderMod, clauses.privateVars,
4034 makeArrayAttr(builder.getContext(), clauses.privateSyms),
4035 clauses.privateNeedsBarrier);
4036}
4037
4038LogicalResult DistributeOp::verify() {
4039 if (this->getDistScheduleChunkSize() && !this->getDistScheduleStatic())
4040 return emitOpError() << "chunk size set without "
4041 "dist_schedule_static being present";
4042
4044 getOperation(), getAllocateVars(), getAllocatorVars(),
4045 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
4046 getPrivateVars(), getPrivateSymsAttr())))
4047 return failure();
4048
4049 if (failed(verifyPrivateVarList(*this)))
4050 return failure();
4051
4052 return success();
4053}
4054
4055LogicalResult DistributeOp::verifyRegions() {
4056 if (LoopWrapperInterface nested = getNestedWrapper()) {
4057 if (!isComposite())
4058 return emitError()
4059 << "'omp.composite' attribute missing from composite wrapper";
4060 // Check for the allowed leaf constructs that may appear in a composite
4061 // construct directly after DISTRIBUTE.
4062 if (isa<WsloopOp>(nested)) {
4063 Operation *parentOp = (*this)->getParentOp();
4064 if (!llvm::dyn_cast_if_present<ParallelOp>(parentOp) ||
4065 !cast<ComposableOpInterface>(parentOp).isComposite()) {
4066 return emitError() << "an 'omp.wsloop' nested wrapper is only allowed "
4067 "when a composite 'omp.parallel' is the direct "
4068 "parent";
4069 }
4070 } else if (!isa<SimdOp>(nested))
4071 return emitError() << "only supported nested wrappers are 'omp.simd' and "
4072 "'omp.wsloop'";
4073 } else if (isComposite()) {
4074 return emitError()
4075 << "'omp.composite' attribute present in non-composite wrapper";
4076 }
4077
4078 return success();
4079}
4080
4081//===----------------------------------------------------------------------===//
4082// DeclareMapperOp / DeclareMapperInfoOp
4083//===----------------------------------------------------------------------===//
4084
4085void DeclareMapperInfoOp::build(OpBuilder &builder, OperationState &state,
4086 const DeclareMapperInfoOperands &clauses) {
4087 DeclareMapperInfoOp::build(builder, state, clauses.mapVars,
4088 clauses.mapIterated);
4089}
4090
4091LogicalResult DeclareMapperInfoOp::verify() {
4092 return verifyMapClause(*this, getMapVars(), getMapIterated());
4093}
4094
4095LogicalResult DeclareMapperOp::verifyRegions() {
4096 if (!llvm::isa_and_present<DeclareMapperInfoOp>(
4097 getRegion().getBlocks().front().getTerminator()))
4098 return emitOpError() << "expected terminator to be a DeclareMapperInfoOp";
4099
4100 return success();
4101}
4102
4103//===----------------------------------------------------------------------===//
4104// DeclareReductionOp
4105//===----------------------------------------------------------------------===//
4106
4107LogicalResult DeclareReductionOp::verifyRegions() {
4108 if (!getAllocRegion().empty()) {
4109 for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
4110 if (yieldOp.getResults().size() != 1 ||
4111 yieldOp.getResults().getTypes()[0] != getType())
4112 return emitOpError() << "expects alloc region to yield a value "
4113 "of the reduction type";
4114 }
4115 }
4116
4117 if (getInitializerRegion().empty())
4118 return emitOpError() << "expects non-empty initializer region";
4119 Block &initializerEntryBlock = getInitializerRegion().front();
4120
4121 if (initializerEntryBlock.getNumArguments() == 1) {
4122 if (!getAllocRegion().empty())
4123 return emitOpError() << "expects two arguments to the initializer region "
4124 "when an allocation region is used";
4125 } else if (initializerEntryBlock.getNumArguments() == 2) {
4126 if (getAllocRegion().empty())
4127 return emitOpError() << "expects one argument to the initializer region "
4128 "when no allocation region is used";
4129 } else {
4130 return emitOpError()
4131 << "expects one or two arguments to the initializer region";
4132 }
4133
4134 for (mlir::Value arg : initializerEntryBlock.getArguments())
4135 if (arg.getType() != getType())
4136 return emitOpError() << "expects initializer region argument to match "
4137 "the reduction type";
4138
4139 for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
4140 if (yieldOp.getResults().size() != 1 ||
4141 yieldOp.getResults().getTypes()[0] != getType())
4142 return emitOpError() << "expects initializer region to yield a value "
4143 "of the reduction type";
4144 }
4145
4146 if (getReductionRegion().empty())
4147 return emitOpError() << "expects non-empty reduction region";
4148 Block &reductionEntryBlock = getReductionRegion().front();
4149 if (reductionEntryBlock.getNumArguments() != 2 ||
4150 reductionEntryBlock.getArgumentTypes()[0] !=
4151 reductionEntryBlock.getArgumentTypes()[1] ||
4152 reductionEntryBlock.getArgumentTypes()[0] != getType())
4153 return emitOpError() << "expects reduction region with two arguments of "
4154 "the reduction type";
4155 for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
4156 if (yieldOp.getResults().size() != 1 ||
4157 yieldOp.getResults().getTypes()[0] != getType())
4158 return emitOpError() << "expects reduction region to yield a value "
4159 "of the reduction type";
4160 }
4161
4162 if (!getAtomicReductionRegion().empty()) {
4163 Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();
4164 if (atomicReductionEntryBlock.getNumArguments() != 2 ||
4165 atomicReductionEntryBlock.getArgumentTypes()[0] !=
4166 atomicReductionEntryBlock.getArgumentTypes()[1])
4167 return emitOpError() << "expects atomic reduction region with two "
4168 "arguments of the same type";
4169 auto ptrType = llvm::dyn_cast<PointerLikeType>(
4170 atomicReductionEntryBlock.getArgumentTypes()[0]);
4171 if (!ptrType ||
4172 (ptrType.getElementType() && ptrType.getElementType() != getType()))
4173 return emitOpError() << "expects atomic reduction region arguments to "
4174 "be accumulators containing the reduction type";
4175 }
4176
4177 if (getCleanupRegion().empty())
4178 return success();
4179 Block &cleanupEntryBlock = getCleanupRegion().front();
4180 if (cleanupEntryBlock.getNumArguments() != 1 ||
4181 cleanupEntryBlock.getArgument(0).getType() != getType())
4182 return emitOpError() << "expects cleanup region with one argument "
4183 "of the reduction type";
4184
4185 return success();
4186}
4187
4188//===----------------------------------------------------------------------===//
4189// TaskOp
4190//===----------------------------------------------------------------------===//
4191
4192void TaskOp::build(OpBuilder &builder, OperationState &state,
4193 const TaskOperands &clauses) {
4194 MLIRContext *ctx = builder.getContext();
4195 TaskOp::build(builder, state, clauses.iterated, clauses.affinityVars,
4196 clauses.allocateVars, clauses.allocatorVars,
4197 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
4198 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
4199 makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,
4200 makeArrayAttr(ctx, clauses.dependIteratedKinds),
4201 clauses.dependIterated, clauses.final, clauses.ifExpr,
4202 clauses.inReductionVars,
4203 makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
4204 makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
4205 clauses.priority, /*private_vars=*/clauses.privateVars,
4206 /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),
4207 clauses.privateNeedsBarrier, clauses.threadset, clauses.untied,
4208 clauses.eventHandle);
4209}
4210
4211LogicalResult TaskOp::verify() {
4213 getOperation(), getAllocateVars(), getAllocatorVars(),
4214 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
4215 getPrivateVars(), getPrivateSymsAttr())))
4216 return failure();
4217
4218 LogicalResult verifyDependVars =
4219 verifyDependVarList(*this, getDependKinds(), getDependVars(),
4220 getDependIteratedKinds(), getDependIterated());
4221 if (failed(verifyDependVars))
4222 return verifyDependVars;
4223
4224 if (failed(verifyPrivateVarList(*this)))
4225 return failure();
4226
4227 return verifyReductionVarList(*this, getInReductionSyms(),
4228 getInReductionVars(), getInReductionByref());
4229}
4230
4231//===----------------------------------------------------------------------===//
4232// TaskgroupOp
4233//===----------------------------------------------------------------------===//
4234
4235void TaskgroupOp::build(OpBuilder &builder, OperationState &state,
4236 const TaskgroupOperands &clauses) {
4237 MLIRContext *ctx = builder.getContext();
4238 TaskgroupOp::build(builder, state, clauses.allocateVars,
4239 clauses.allocatorVars,
4240 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
4241 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
4242 clauses.taskReductionVars,
4243 makeDenseBoolArrayAttr(ctx, clauses.taskReductionByref),
4244 makeArrayAttr(ctx, clauses.taskReductionSyms),
4245 /*task_reduction_assume_bounded_recipe_effects=*/nullptr);
4246}
4247
4248LogicalResult TaskgroupOp::verify() {
4250 getOperation(), getAllocateVars(), getAllocatorVars(),
4251 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr())))
4252 return failure();
4253
4254 return verifyReductionVarList(*this, getTaskReductionSyms(),
4255 getTaskReductionVars(),
4256 getTaskReductionByref());
4257}
4258
4261 auto iface = dyn_cast<BlockArgOpenMPOpInterface>(op);
4262 auto taskVars = iface.getTaskReductionVars();
4263 for (auto &source : MutableArrayRef(taskVars.getBase(), taskVars.size())) {
4264 effects.emplace_back(MemoryEffects::Read::get(), &source);
4265 effects.emplace_back(MemoryEffects::Write::get(), &source);
4266 }
4267
4268 for (BlockArgument arg : iface.getTaskReductionBlockArgs()) {
4269 effects.emplace_back(MemoryEffects::Read::get(), arg);
4270 effects.emplace_back(MemoryEffects::Write::get(), arg);
4271 }
4272
4273 // The task_reduction clause does not necessarily allocate or free private
4274 // accumulators. We avoid attaching an allocate effect to these accumulators
4275 // as this would incorrectly imply they are fresh storage.
4276 if (!taskVars.empty()) {
4277 effects.emplace_back(MemoryEffects::Allocate::get());
4278 effects.emplace_back(MemoryEffects::Free::get());
4279 }
4280}
4281
4284 &effects) {
4285 effects.emplace_back(MemoryEffects::Read::get());
4286 effects.emplace_back(MemoryEffects::Write::get());
4287}
4288
4289void TaskgroupOp::getEffects(
4291 &effects) {
4293
4294 if (!getTaskReductionVars().empty() &&
4295 !getTaskReductionAssumeBoundedRecipeEffects())
4296 addUnknownMemoryEffects(effects);
4297 else
4298 getTaskReductionEffects(*this, effects);
4299}
4300
4301//===----------------------------------------------------------------------===//
4302// TaskloopContextOp
4303//===----------------------------------------------------------------------===//
4304
4305void TaskloopContextOp::build(OpBuilder &builder, OperationState &state,
4306 const TaskloopContextOperands &clauses) {
4307 MLIRContext *ctx = builder.getContext();
4308 TaskloopContextOp::build(
4309 builder, state, clauses.allocateVars, clauses.allocatorVars,
4310 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
4311 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices), clauses.final,
4312 clauses.grainsizeMod, clauses.grainsize, clauses.ifExpr,
4313 clauses.inReductionVars,
4314 makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
4315 makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
4316 clauses.nogroup, clauses.numTasksMod, clauses.numTasks, clauses.priority,
4317 /*private_vars=*/clauses.privateVars,
4318 /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),
4319 clauses.privateNeedsBarrier, makeArrayAttr(ctx, clauses.reductionMods),
4320 clauses.reductionVars,
4321 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
4322 makeArrayAttr(ctx, clauses.reductionSyms), clauses.threadset,
4323 clauses.untied);
4324 state.addAttribute("omp.combined", UnitAttr::get(ctx));
4325}
4326
4327TaskloopWrapperOp TaskloopContextOp::getLoopOp() {
4328 return cast<TaskloopWrapperOp>(
4329 *llvm::find_if(getRegion().front(), [](mlir::Operation &op) {
4330 return isa<TaskloopWrapperOp>(op);
4331 }));
4332}
4333
4334LogicalResult TaskloopContextOp::verify() {
4335 if (failed(verifyPrivateVarList(*this)))
4336 return failure();
4338 getOperation(), getAllocateVars(), getAllocatorVars(),
4339 getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
4340 getPrivateVars(), getPrivateSymsAttr())))
4341 return failure();
4342
4343 if (failed(verifyReductionVarList(*this, getReductionSyms(),
4344 getReductionVars(), getReductionByref(),
4345 getReductionModsAttr())) ||
4346 failed(verifyReductionVarList(*this, getInReductionSyms(),
4347 getInReductionVars(),
4348 getInReductionByref())))
4349 return failure();
4350
4351 if (!getReductionVars().empty() && getNogroup())
4352 return emitError("if a reduction clause is present on the taskloop "
4353 "directive, the nogroup clause must not be specified");
4354 for (auto var : getReductionVars()) {
4355 if (llvm::is_contained(getInReductionVars(), var))
4356 return emitError("the same list item cannot appear in both a reduction "
4357 "and an in_reduction clause");
4358 }
4359
4360 if (getGrainsize() && getNumTasks()) {
4361 return emitError(
4362 "the grainsize clause and num_tasks clause are mutually exclusive and "
4363 "may not appear on the same taskloop directive");
4364 }
4365
4366 // Without this restriction, any compound construct including `taskloop` would
4367 // fail to correctly identify the whole chain of operations (see
4368 // ComposableOpInterface::findCapturedOp()), as well as failing to do so even
4369 // for standalone `taskloop` constructs.
4370 if (!isCombined())
4371 return emitOpError("must always contain the 'omp.combined' attribute");
4372
4373 return success();
4374}
4375
4376LogicalResult TaskloopContextOp::verifyRegions() {
4377 Region &region = getRegion();
4378 auto loopWrapperIt = llvm::find_if(region.front(), [](mlir::Operation &op) {
4379 return isa<TaskloopWrapperOp>(op);
4380 });
4381 if (loopWrapperIt == region.front().end())
4382 return emitOpError()
4383 << "expected a TaskloopWrapperOp directly nested in the region";
4384
4385 auto loopWrapperOp = cast<TaskloopWrapperOp>(*loopWrapperIt);
4386 auto loopNestOp = dyn_cast<LoopNestOp>(loopWrapperOp.getWrappedLoop());
4387 // This will fail the verifier for TaskloopWrapperOp and print an error
4388 // message there.
4389 if (!loopNestOp)
4390 return failure();
4391
4392 std::function<bool(Value)> isValidBoundValue = [&](Value value) -> bool {
4393 Region *valueRegion = value.getParentRegion();
4394 // A loop bound value defined outside of the taskloop context region is
4395 // valid. A region is considered an ancestor of itself.
4396 if (!region.isAncestor(valueRegion))
4397 return true;
4398
4399 Operation *defOp = value.getDefiningOp();
4400 if (!defOp || defOp->getNumRegions() != 0 || !isPure(defOp))
4401 return false;
4402
4403 return llvm::all_of(defOp->getOperands(), isValidBoundValue);
4404 };
4405 auto hasUnsupportedTaskloopLocalBound = [&](OperandRange range) -> bool {
4406 return llvm::any_of(range,
4407 [&](Value value) { return !isValidBoundValue(value); });
4408 };
4409
4410 if (hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopLowerBounds()) ||
4411 hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopUpperBounds()) ||
4412 hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopSteps())) {
4413 return emitOpError()
4414 << "expects loop bounds and steps to be defined outside of the "
4415 "taskloop.context region or by pure, regionless operations "
4416 "that do not depend on block arguments";
4417 }
4418
4419 return success();
4420}
4421
4422//===----------------------------------------------------------------------===//
4423// TaskloopWrapperOp
4424//===----------------------------------------------------------------------===//
4425
4426void TaskloopWrapperOp::build(OpBuilder &builder, OperationState &state,
4427 const TaskloopWrapperOperands &clauses) {
4428 TaskloopWrapperOp::build(builder, state);
4429}
4430
4431TaskloopContextOp TaskloopWrapperOp::getTaskloopContext() {
4432 return dyn_cast<TaskloopContextOp>(getOperation()->getParentOp());
4433}
4434
4435LogicalResult TaskloopWrapperOp::verify() {
4436 TaskloopContextOp context = getTaskloopContext();
4437 if (!context)
4438 return emitOpError() << "expected to be nested in a taskloop context op";
4439 return success();
4440}
4441
4442LogicalResult TaskloopWrapperOp::verifyRegions() {
4443 if (LoopWrapperInterface nested = getNestedWrapper()) {
4444 if (!isComposite())
4445 return emitError()
4446 << "'omp.composite' attribute missing from composite wrapper";
4447
4448 // Check for the allowed leaf constructs that may appear in a composite
4449 // construct directly after TASKLOOP.
4450 if (!isa<SimdOp>(nested))
4451 return emitError() << "only supported nested wrapper is 'omp.simd'";
4452 } else if (isComposite()) {
4453 return emitError()
4454 << "'omp.composite' attribute present in non-composite wrapper";
4455 }
4456
4457 return success();
4458}
4459
4460//===----------------------------------------------------------------------===//
4461// LoopNestOp
4462//===----------------------------------------------------------------------===//
4463
4464ParseResult LoopNestOp::parse(OpAsmParser &parser, OperationState &result) {
4465 // Parse an opening `(` followed by induction variables followed by `)`
4468 Type loopVarType;
4470 parser.parseColonType(loopVarType) ||
4471 // Parse loop bounds.
4472 parser.parseEqual() ||
4473 parser.parseOperandList(lbs, ivs.size(), OpAsmParser::Delimiter::Paren) ||
4474 parser.parseKeyword("to") ||
4475 parser.parseOperandList(ubs, ivs.size(), OpAsmParser::Delimiter::Paren))
4476 return failure();
4477
4478 for (auto &iv : ivs)
4479 iv.type = loopVarType;
4480
4481 auto *ctx = parser.getBuilder().getContext();
4482 // Parse "inclusive" flag.
4483 if (succeeded(parser.parseOptionalKeyword("inclusive")))
4484 result.addAttribute("loop_inclusive", UnitAttr::get(ctx));
4485
4486 // Parse step values.
4488 if (parser.parseKeyword("step") ||
4489 parser.parseOperandList(steps, ivs.size(), OpAsmParser::Delimiter::Paren))
4490 return failure();
4491
4492 // Parse collapse
4493 int64_t value = 0;
4494 if (!parser.parseOptionalKeyword("collapse") &&
4495 (parser.parseLParen() || parser.parseInteger(value) ||
4496 parser.parseRParen()))
4497 return failure();
4498 if (value > 1)
4499 result.addAttribute(
4500 "collapse_num_loops",
4501 IntegerAttr::get(parser.getBuilder().getI64Type(), value));
4502
4503 // Parse tiles
4505 auto parseTiles = [&]() -> ParseResult {
4506 int64_t tile;
4507 if (parser.parseInteger(tile))
4508 return failure();
4509 tiles.push_back(tile);
4510 return success();
4511 };
4512
4513 if (!parser.parseOptionalKeyword("tiles") &&
4514 (parser.parseLParen() || parser.parseCommaSeparatedList(parseTiles) ||
4515 parser.parseRParen()))
4516 return failure();
4517
4518 if (tiles.size() > 0)
4519 result.addAttribute("tile_sizes", DenseI64ArrayAttr::get(ctx, tiles));
4520
4521 // Parse the body.
4522 Region *region = result.addRegion();
4523 if (parser.parseRegion(*region, ivs))
4524 return failure();
4525
4526 // Resolve operands.
4527 if (parser.resolveOperands(lbs, loopVarType, result.operands) ||
4528 parser.resolveOperands(ubs, loopVarType, result.operands) ||
4529 parser.resolveOperands(steps, loopVarType, result.operands))
4530 return failure();
4531
4532 // Parse the optional attribute list.
4533 return parser.parseOptionalAttrDict(result.attributes);
4534}
4535
4536void LoopNestOp::print(OpAsmPrinter &p) {
4537 Region &region = getRegion();
4538 auto args = region.getArguments();
4539 p << " (" << args << ") : " << args[0].getType() << " = ("
4540 << getLoopLowerBounds() << ") to (" << getLoopUpperBounds() << ") ";
4541 if (getLoopInclusive())
4542 p << "inclusive ";
4543 p << "step (" << getLoopSteps() << ") ";
4544 if (int64_t numCollapse = getCollapseNumLoops())
4545 if (numCollapse > 1)
4546 p << "collapse(" << numCollapse << ") ";
4547
4548 if (const auto tiles = getTileSizes())
4549 p << "tiles(" << tiles.value() << ") ";
4550
4551 p.printRegion(region, /*printEntryBlockArgs=*/false);
4552}
4553
4554void LoopNestOp::build(OpBuilder &builder, OperationState &state,
4555 const LoopNestOperands &clauses) {
4556 MLIRContext *ctx = builder.getContext();
4557 LoopNestOp::build(builder, state, clauses.collapseNumLoops,
4558 clauses.loopLowerBounds, clauses.loopUpperBounds,
4559 clauses.loopSteps, clauses.loopInclusive,
4560 makeDenseI64ArrayAttr(ctx, clauses.tileSizes));
4561}
4562
4563LogicalResult LoopNestOp::verify() {
4564 if (getLoopLowerBounds().empty())
4565 return emitOpError() << "must represent at least one loop";
4566
4567 if (getLoopLowerBounds().size() != getIVs().size())
4568 return emitOpError() << "number of range arguments and IVs do not match";
4569
4570 for (auto [lb, iv] : llvm::zip_equal(getLoopLowerBounds(), getIVs())) {
4571 if (lb.getType() != iv.getType())
4572 return emitOpError()
4573 << "range argument type does not match corresponding IV type";
4574 }
4575
4576 uint64_t numIVs = getIVs().size();
4577
4578 if (const auto &numCollapse = getCollapseNumLoops())
4579 if (numCollapse > numIVs)
4580 return emitOpError()
4581 << "collapse value is larger than the number of loops";
4582
4583 if (const auto &tiles = getTileSizes())
4584 if (tiles.value().size() > numIVs)
4585 return emitOpError() << "too few canonical loops for tile dimensions";
4586
4587 if (!llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp()))
4588 return emitOpError() << "expects parent op to be a loop wrapper";
4589
4590 return success();
4591}
4592
4593void LoopNestOp::gatherWrappers(
4595 Operation *parent = (*this)->getParentOp();
4596 while (auto wrapper =
4597 llvm::dyn_cast_if_present<LoopWrapperInterface>(parent)) {
4598 wrappers.push_back(wrapper);
4599 parent = parent->getParentOp();
4600 }
4601}
4602
4603//===----------------------------------------------------------------------===//
4604// OpenMP canonical loop handling
4605//===----------------------------------------------------------------------===//
4606
4607std::tuple<NewCliOp, OpOperand *, OpOperand *>
4608mlir::omp ::decodeCli(Value cli) {
4609
4610 // Defining a CLI for a generated loop is optional; if there is none then
4611 // there is no followup-tranformation
4612 if (!cli)
4613 return {{}, nullptr, nullptr};
4614
4615 assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&
4616 "Unexpected type of cli");
4617
4618 NewCliOp create = cast<NewCliOp>(cli.getDefiningOp());
4619 OpOperand *gen = nullptr;
4620 OpOperand *cons = nullptr;
4621 for (OpOperand &use : cli.getUses()) {
4622 auto op = cast<LoopTransformationInterface>(use.getOwner());
4623
4624 unsigned opnum = use.getOperandNumber();
4625 if (op.isGeneratee(opnum)) {
4626 assert(!gen && "Each CLI may have at most one def");
4627 gen = &use;
4628 } else if (op.isApplyee(opnum)) {
4629 assert(!cons && "Each CLI may have at most one consumer");
4630 cons = &use;
4631 } else {
4632 llvm_unreachable("Unexpected operand for a CLI");
4633 }
4634 }
4635
4636 return {create, gen, cons};
4637}
4638
4639ClauseProcBindKind
4640mlir::omp::convertProcBindKind(llvm::omp::ProcBindKind kind) {
4641 switch (kind) {
4642 case llvm::omp::ProcBindKind::OMP_PROC_BIND_close:
4643 return ClauseProcBindKind::Close;
4644 case llvm::omp::ProcBindKind::OMP_PROC_BIND_master:
4645 return ClauseProcBindKind::Master;
4646 case llvm::omp::ProcBindKind::OMP_PROC_BIND_primary:
4647 return ClauseProcBindKind::Primary;
4648 case llvm::omp::ProcBindKind::OMP_PROC_BIND_spread:
4649 return ClauseProcBindKind::Spread;
4650 case llvm::omp::ProcBindKind::OMP_PROC_BIND_default:
4651 case llvm::omp::ProcBindKind::OMP_PROC_BIND_unknown:
4652 break;
4653 }
4654 llvm_unreachable("unexpected proc-bind kind");
4655}
4656
4657void NewCliOp::build(::mlir::OpBuilder &odsBuilder,
4658 ::mlir::OperationState &odsState) {
4659 odsState.addTypes(CanonicalLoopInfoType::get(odsBuilder.getContext()));
4660}
4661
4662void NewCliOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {
4663 Value result = getResult();
4664 auto [newCli, gen, cons] = decodeCli(result);
4665
4666 // Structured binding `gen` cannot be captured in lambdas before C++20
4667 OpOperand *generator = gen;
4668
4669 // Derive the CLI variable name from its generator:
4670 // * "canonloop" for omp.canonical_loop
4671 // * custom name for loop transformation generatees
4672 // * "cli" as fallback if no generator
4673 // * "_r<idx>" suffix for nested loops, where <idx> is the sequential order
4674 // at that level
4675 // * "_s<idx>" suffix for operations with multiple regions, where <idx> is
4676 // the index of that region
4677 std::string cliName{"cli"};
4678 if (gen) {
4679 cliName =
4681 .Case([&](CanonicalLoopOp op) {
4682 return generateLoopNestingName("canonloop", op);
4683 })
4684 .Case([&](UnrollHeuristicOp op) -> std::string {
4685 llvm_unreachable("heuristic unrolling does not generate a loop");
4686 })
4687 .Case([&](FuseOp op) -> std::string {
4688 unsigned opnum = generator->getOperandNumber();
4689 // The position of the first loop to be fused is the same position
4690 // as the resulting fused loop
4691 if (op.getFirst().has_value() && opnum != op.getFirst().value())
4692 return "canonloop_fuse";
4693 else
4694 return "fused";
4695 })
4696 .Case([&](TileOp op) -> std::string {
4697 auto [generateesFirst, generateesCount] =
4698 op.getGenerateesODSOperandIndexAndLength();
4699 unsigned firstGrid = generateesFirst;
4700 unsigned firstIntratile = generateesFirst + generateesCount / 2;
4701 unsigned end = generateesFirst + generateesCount;
4702 unsigned opnum = generator->getOperandNumber();
4703 // In the OpenMP apply and looprange clauses, indices are 1-based
4704 if (firstGrid <= opnum && opnum < firstIntratile) {
4705 unsigned gridnum = opnum - firstGrid + 1;
4706 return ("grid" + Twine(gridnum)).str();
4707 }
4708 if (firstIntratile <= opnum && opnum < end) {
4709 unsigned intratilenum = opnum - firstIntratile + 1;
4710 return ("intratile" + Twine(intratilenum)).str();
4711 }
4712 llvm_unreachable("Unexpected generatee argument");
4713 })
4714 .DefaultUnreachable("TODO: Custom name for this operation");
4715 }
4716
4717 setNameFn(result, cliName);
4718}
4719
4720LogicalResult NewCliOp::verify() {
4721 Value cli = getResult();
4722
4723 assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&
4724 "Unexpected type of cli");
4725
4726 // Check that the CLI is used in at most generator and one consumer
4727 OpOperand *gen = nullptr;
4728 OpOperand *cons = nullptr;
4729 for (mlir::OpOperand &use : cli.getUses()) {
4730 auto op = dyn_cast<mlir::omp::LoopTransformationInterface>(use.getOwner());
4731 if (!op) {
4732 InFlightDiagnostic error =
4733 emitOpError("CLI must only be used by loop transformations");
4734 error.attachNote(use.getOwner()->getLoc()).append("invalid use here:");
4735 return error;
4736 }
4737
4738 unsigned opnum = use.getOperandNumber();
4739 if (op.isGeneratee(opnum)) {
4740 if (gen) {
4741 InFlightDiagnostic error =
4742 emitOpError("CLI must have at most one generator");
4743 error.attachNote(gen->getOwner()->getLoc())
4744 .append("first generator here:");
4745 error.attachNote(use.getOwner()->getLoc())
4746 .append("second generator here:");
4747 return error;
4748 }
4749
4750 gen = &use;
4751 } else if (op.isApplyee(opnum)) {
4752 if (cons) {
4753 InFlightDiagnostic error =
4754 emitOpError("CLI must have at most one consumer");
4755 error.attachNote(cons->getOwner()->getLoc())
4756 .append("first consumer here:")
4757 .appendOp(*cons->getOwner(),
4758 OpPrintingFlags().printGenericOpForm());
4759 error.attachNote(use.getOwner()->getLoc())
4760 .append("second consumer here:")
4761 .appendOp(*use.getOwner(), OpPrintingFlags().printGenericOpForm());
4762 return error;
4763 }
4764
4765 cons = &use;
4766 } else {
4767 llvm_unreachable("Unexpected operand for a CLI");
4768 }
4769 }
4770
4771 // If the CLI is source of a transformation, it must have a generator
4772 if (cons && !gen) {
4773 InFlightDiagnostic error = emitOpError("CLI has no generator");
4774 error.attachNote(cons->getOwner()->getLoc())
4775 .append("see consumer here: ")
4776 .appendOp(*cons->getOwner(), OpPrintingFlags().printGenericOpForm());
4777 return error;
4778 }
4779
4780 return success();
4781}
4782
4783void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
4784 Value tripCount) {
4785 odsState.addOperands(tripCount);
4786 odsState.addOperands(Value());
4787 (void)odsState.addRegion();
4788}
4789
4790void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
4791 Value tripCount, ::mlir::Value cli) {
4792 odsState.addOperands(tripCount);
4793 odsState.addOperands(cli);
4794 (void)odsState.addRegion();
4795}
4796
4797void CanonicalLoopOp::getAsmBlockNames(OpAsmSetBlockNameFn setNameFn) {
4798 setNameFn(&getRegion().front(), "body_entry");
4799}
4800
4801void CanonicalLoopOp::getAsmBlockArgumentNames(Region &region,
4802 OpAsmSetValueNameFn setNameFn) {
4803 std::string ivName = generateLoopNestingName("iv", *this);
4804 setNameFn(region.getArgument(0), ivName);
4805}
4806
4807void CanonicalLoopOp::print(OpAsmPrinter &p) {
4808 if (getCli())
4809 p << '(' << getCli() << ')';
4810 p << ' ' << getInductionVar() << " : " << getInductionVar().getType()
4811 << " in range(" << getTripCount() << ") ";
4812
4813 p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
4814 /*printBlockTerminators=*/true);
4815
4816 p.printOptionalAttrDict((*this)->getDiscardableAttrDictionary().getValue());
4817}
4818
4819mlir::ParseResult CanonicalLoopOp::parse(::mlir::OpAsmParser &parser,
4821 CanonicalLoopInfoType cliType =
4822 CanonicalLoopInfoType::get(parser.getContext());
4823
4824 // Parse (optional) omp.cli identifier
4826 SmallVector<mlir::Value, 1> cliOperand;
4827 if (!parser.parseOptionalLParen()) {
4828 if (parser.parseOperand(cli) ||
4829 parser.resolveOperand(cli, cliType, cliOperand) || parser.parseRParen())
4830 return failure();
4831 }
4832
4833 // We derive the type of tripCount from inductionVariable. MLIR requires the
4834 // type of tripCount to be known when calling resolveOperand so we have parse
4835 // the type before processing the inductionVariable.
4836 OpAsmParser::Argument inductionVariable;
4838 if (parser.parseArgument(inductionVariable, /*allowType*/ true) ||
4839 parser.parseKeyword("in") || parser.parseKeyword("range") ||
4840 parser.parseLParen() || parser.parseOperand(tripcount) ||
4841 parser.parseRParen() ||
4842 parser.resolveOperand(tripcount, inductionVariable.type, result.operands))
4843 return failure();
4844
4845 // Parse the loop body.
4846 Region *region = result.addRegion();
4847 if (parser.parseRegion(*region, {inductionVariable}))
4848 return failure();
4849
4850 // We parsed the cli operand forst, but because it is optional, it must be
4851 // last in the operand list.
4852 result.operands.append(cliOperand);
4853
4854 // Parse the optional attribute list.
4855 if (parser.parseOptionalAttrDict(result.attributes))
4856 return failure();
4857
4858 return mlir::success();
4859}
4860
4861LogicalResult CanonicalLoopOp::verify() {
4862 // The region's entry must accept the induction variable
4863 // It can also be empty if just created
4864 if (!getRegion().empty()) {
4865 Region &region = getRegion();
4866 if (region.getNumArguments() != 1)
4867 return emitOpError(
4868 "Canonical loop region must have exactly one argument");
4869
4870 if (getInductionVar().getType() != getTripCount().getType())
4871 return emitOpError(
4872 "Region argument must be the same type as the trip count");
4873 }
4874
4875 return success();
4876}
4877
4878Value CanonicalLoopOp::getInductionVar() { return getRegion().getArgument(0); }
4879
4880std::pair<unsigned, unsigned>
4881CanonicalLoopOp::getApplyeesODSOperandIndexAndLength() {
4882 // No applyees
4883 return {0, 0};
4884}
4885
4886std::pair<unsigned, unsigned>
4887CanonicalLoopOp::getGenerateesODSOperandIndexAndLength() {
4888 return getODSOperandIndexAndLength(odsIndex_cli);
4889}
4890
4891//===----------------------------------------------------------------------===//
4892// UnrollHeuristicOp
4893//===----------------------------------------------------------------------===//
4894
4895void UnrollHeuristicOp::build(::mlir::OpBuilder &odsBuilder,
4896 ::mlir::OperationState &odsState,
4897 ::mlir::Value cli) {
4898 odsState.addOperands(cli);
4899}
4900
4901void UnrollHeuristicOp::print(OpAsmPrinter &p) {
4902 p << '(' << getApplyee() << ')';
4903
4904 p.printOptionalAttrDict((*this)->getDiscardableAttrDictionary().getValue());
4905}
4906
4907mlir::ParseResult UnrollHeuristicOp::parse(::mlir::OpAsmParser &parser,
4909 auto cliType = CanonicalLoopInfoType::get(parser.getContext());
4910
4911 if (parser.parseLParen())
4912 return failure();
4913
4915 if (parser.parseOperand(applyee) ||
4916 parser.resolveOperand(applyee, cliType, result.operands))
4917 return failure();
4918
4919 if (parser.parseRParen())
4920 return failure();
4921
4922 // Optional output loop (full unrolling has none)
4923 if (!parser.parseOptionalArrow()) {
4924 if (parser.parseLParen() || parser.parseRParen())
4925 return failure();
4926 }
4927
4928 // Parse the optional attribute list.
4929 if (parser.parseOptionalAttrDict(result.attributes))
4930 return failure();
4931
4932 return mlir::success();
4933}
4934
4935std::pair<unsigned, unsigned>
4936UnrollHeuristicOp ::getApplyeesODSOperandIndexAndLength() {
4937 return getODSOperandIndexAndLength(odsIndex_applyee);
4938}
4939
4940std::pair<unsigned, unsigned>
4941UnrollHeuristicOp::getGenerateesODSOperandIndexAndLength() {
4942 return {0, 0};
4943}
4944
4945//===----------------------------------------------------------------------===//
4946// UnrollFullOp
4947//===----------------------------------------------------------------------===//
4948
4949void UnrollFullOp::build(::mlir::OpBuilder &odsBuilder,
4950 ::mlir::OperationState &odsState, ::mlir::Value cli) {
4951 odsState.addOperands(cli);
4952}
4953
4954void UnrollFullOp::print(OpAsmPrinter &p) {
4955 p << '(' << getApplyee() << ')';
4956
4957 p.printOptionalAttrDict((*this)->getDiscardableAttrDictionary().getValue());
4958}
4959
4960mlir::ParseResult UnrollFullOp::parse(::mlir::OpAsmParser &parser,
4962 auto cliType = CanonicalLoopInfoType::get(parser.getContext());
4963
4964 if (parser.parseLParen())
4965 return failure();
4966
4968 if (parser.parseOperand(applyee) ||
4969 parser.resolveOperand(applyee, cliType, result.operands))
4970 return failure();
4971
4972 if (parser.parseRParen())
4973 return failure();
4974
4975 // Optional output loop; full unrolling has none.
4976 if (!parser.parseOptionalArrow()) {
4977 if (parser.parseLParen() || parser.parseRParen())
4978 return failure();
4979 }
4980
4981 // Parse the optional attribute list.
4982 if (parser.parseOptionalAttrDict(result.attributes))
4983 return failure();
4984
4985 return mlir::success();
4986}
4987
4988std::pair<unsigned, unsigned>
4989UnrollFullOp::getApplyeesODSOperandIndexAndLength() {
4990 return getODSOperandIndexAndLength(odsIndex_applyee);
4991}
4992
4993std::pair<unsigned, unsigned>
4994UnrollFullOp::getGenerateesODSOperandIndexAndLength() {
4995 return {0, 0};
4996}
4997
4998LogicalResult UnrollFullOp::verify() {
4999 auto [create, gen, cons] = decodeCli(getApplyee());
5000 if (!gen)
5001 return emitOpError() << "applyee CLI has no generator";
5002
5003 // Full unrolling leaves no loop, so the trip count must be constant. Only
5004 // omp.canonical_loop states one.
5005 if (auto loop = dyn_cast<CanonicalLoopOp>(gen->getOwner())) {
5006 if (!matchPattern(loop.getTripCount(), m_Constant()))
5007 return emitOpError() << "applyee loop must have a constant trip count";
5008 }
5009
5010 return success();
5011}
5012
5013//===----------------------------------------------------------------------===//
5014// UnrollPartialOp
5015//===----------------------------------------------------------------------===//
5016
5017void UnrollPartialOp::build(::mlir::OpBuilder &odsBuilder,
5019 uint64_t unrollFactor) {
5020 odsState.addOperands(cli);
5021 Properties &props = odsState.getOrAddProperties<Properties>();
5022 props.unroll_factor = odsBuilder.getI64IntegerAttr(unrollFactor);
5023}
5024
5025void UnrollPartialOp::print(OpAsmPrinter &p) {
5026 p << '(' << getApplyee() << ')';
5027
5028 SmallVector<NamedAttribute> attrs((*this)->getDiscardableAttrs());
5029 attrs.emplace_back(getUnrollFactorAttrName(), getUnrollFactorAttr());
5030 llvm::sort(attrs);
5031 p.printOptionalAttrDict(attrs);
5032}
5033
5034mlir::ParseResult UnrollPartialOp::parse(::mlir::OpAsmParser &parser,
5036 auto cliType = CanonicalLoopInfoType::get(parser.getContext());
5037
5038 if (parser.parseLParen())
5039 return failure();
5040
5042 if (parser.parseOperand(applyee) ||
5043 parser.resolveOperand(applyee, cliType, result.operands))
5044 return failure();
5045
5046 if (parser.parseRParen())
5047 return failure();
5048
5049 // The unroll factor is carried by the `unroll_factor` attribute.
5050 if (parser.parseOptionalAttrDict(result.attributes))
5051 return failure();
5052
5053 return mlir::success();
5054}
5055
5056std::pair<unsigned, unsigned>
5057UnrollPartialOp::getApplyeesODSOperandIndexAndLength() {
5058 return getODSOperandIndexAndLength(odsIndex_applyee);
5059}
5060
5061std::pair<unsigned, unsigned>
5062UnrollPartialOp::getGenerateesODSOperandIndexAndLength() {
5063 return {0, 0};
5064}
5065
5066//===----------------------------------------------------------------------===//
5067// TileOp
5068//===----------------------------------------------------------------------===//
5069
5070static void printLoopTransformClis(OpAsmPrinter &p, TileOp op,
5071 OperandRange generatees,
5072 OperandRange applyees) {
5073 if (!generatees.empty())
5074 p << '(' << llvm::interleaved(generatees) << ')';
5075
5076 if (!applyees.empty())
5077 p << " <- (" << llvm::interleaved(applyees) << ')';
5078}
5079
5080static ParseResult parseLoopTransformClis(
5081 OpAsmParser &parser,
5084 if (parser.parseOptionalLess()) {
5085 // Syntax 1: generatees present
5086
5087 if (parser.parseOperandList(generateesOperands,
5089 return failure();
5090
5091 if (parser.parseLess())
5092 return failure();
5093 } else {
5094 // Syntax 2: generatees omitted
5095 }
5096
5097 // Parse `<-` (`<` has already been parsed)
5098 if (parser.parseMinus())
5099 return failure();
5100
5101 if (parser.parseOperandList(applyeesOperands,
5103 return failure();
5104
5105 return success();
5106}
5107
5108/// Check properties of the loop nest consisting of the transformation's
5109/// applyees:
5110/// 1. They are nested inside each other
5111/// 2. They are perfectly nested
5112/// (no code with side-effects in-between the loops)
5113/// 3. They are rectangular
5114/// (loop bounds are invariant in respect to the outer loops)
5115///
5116/// TODO: Generalize for LoopTransformationInterface.
5117static LogicalResult checkApplyeesNesting(TileOp op) {
5118 // Collect the loops from the nest
5119 bool isOnlyCanonLoops = true;
5121 for (Value applyee : op.getApplyees()) {
5122 auto [create, gen, cons] = decodeCli(applyee);
5123
5124 if (!gen)
5125 return op.emitOpError() << "applyee CLI has no generator";
5126
5127 auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());
5128 canonLoops.push_back(loop);
5129 if (!loop)
5130 isOnlyCanonLoops = false;
5131 }
5132
5133 // FIXME: We currently can only verify non-rectangularity and perfect nest of
5134 // omp.canonical_loop.
5135 if (!isOnlyCanonLoops)
5136 return success();
5137
5138 DenseSet<Value> parentIVs;
5139 for (auto i : llvm::seq<int>(1, canonLoops.size())) {
5140 auto parentLoop = canonLoops[i - 1];
5141 auto loop = canonLoops[i];
5142
5143 if (parentLoop.getOperation() != loop.getOperation()->getParentOp())
5144 return op.emitOpError()
5145 << "tiled loop nest must be nested within each other";
5146
5147 parentIVs.insert(parentLoop.getInductionVar());
5148
5149 // Canonical loop must be perfectly nested, i.e. the body of the parent must
5150 // only contain the omp.canonical_loop of the nested loops, and
5151 // omp.terminator
5152 bool isPerfectlyNested = [&]() {
5153 auto &parentBody = parentLoop.getRegion();
5154 if (!parentBody.hasOneBlock())
5155 return false;
5156 auto &parentBlock = parentBody.getBlocks().front();
5157
5158 auto nestedLoopIt = parentBlock.begin();
5159 if (nestedLoopIt == parentBlock.end() ||
5160 (&*nestedLoopIt != loop.getOperation()))
5161 return false;
5162
5163 auto termIt = std::next(nestedLoopIt);
5164 if (termIt == parentBlock.end() || !isa<TerminatorOp>(termIt))
5165 return false;
5166
5167 if (std::next(termIt) != parentBlock.end())
5168 return false;
5169
5170 return true;
5171 }();
5172 if (!isPerfectlyNested)
5173 return op.emitOpError() << "tiled loop nest must be perfectly nested";
5174
5175 if (parentIVs.contains(loop.getTripCount()))
5176 return op.emitOpError() << "tiled loop nest must be rectangular";
5177 }
5178
5179 // TODO: The tile sizes must be computed before the loop, but checking this
5180 // requires dominance analysis. For instance:
5181 //
5182 // %canonloop = omp.new_cli
5183 // omp.canonical_loop(%canonloop) %iv : i32 in range(%tc) {
5184 // // write to %x
5185 // omp.terminator
5186 // }
5187 // %ts = llvm.load %x
5188 // omp.tile <- (%canonloop) sizes(%ts : i32)
5189
5190 return success();
5191}
5192
5193LogicalResult TileOp::verify() {
5194 if (getApplyees().empty())
5195 return emitOpError() << "must apply to at least one loop";
5196
5197 if (getSizes().size() != getApplyees().size())
5198 return emitOpError() << "there must be one tile size for each applyee";
5199
5200 if (!getGeneratees().empty() &&
5201 2 * getSizes().size() != getGeneratees().size())
5202 return emitOpError()
5203 << "expecting two times the number of generatees than applyees";
5204
5205 return checkApplyeesNesting(*this);
5206}
5207
5208std::pair<unsigned, unsigned> TileOp ::getApplyeesODSOperandIndexAndLength() {
5209 return getODSOperandIndexAndLength(odsIndex_applyees);
5210}
5211
5212std::pair<unsigned, unsigned> TileOp::getGenerateesODSOperandIndexAndLength() {
5213 return getODSOperandIndexAndLength(odsIndex_generatees);
5214}
5215
5216//===----------------------------------------------------------------------===//
5217// FuseOp
5218//===----------------------------------------------------------------------===//
5219
5220static void printLoopTransformClis(OpAsmPrinter &p, FuseOp op,
5221 OperandRange generatees,
5222 OperandRange applyees) {
5223 if (!generatees.empty())
5224 p << '(' << llvm::interleaved(generatees) << ')';
5225
5226 if (!applyees.empty())
5227 p << " <- (" << llvm::interleaved(applyees) << ')';
5228}
5229
5230LogicalResult FuseOp::verify() {
5231 if (getApplyees().size() < 2)
5232 return emitOpError() << "must apply to at least two loops";
5233
5234 if (getFirst().has_value() && getCount().has_value()) {
5235 int64_t first = getFirst().value();
5236 int64_t count = getCount().value();
5237 if ((unsigned)(first + count - 1) > getApplyees().size())
5238 return emitOpError() << "the numbers of applyees must be at least first "
5239 "minus one plus count attributes";
5240 if (!getGeneratees().empty() &&
5241 getGeneratees().size() != getApplyees().size() + 1 - count)
5242 return emitOpError() << "the number of generatees must be the number of "
5243 "aplyees plus one minus count";
5244
5245 } else {
5246 if (!getGeneratees().empty() && getGeneratees().size() != 1)
5247 return emitOpError()
5248 << "in a complete fuse the number of generatees must be exactly 1";
5249 }
5250 for (auto &&applyee : getApplyees()) {
5251 auto [create, gen, cons] = decodeCli(applyee);
5252
5253 if (!gen)
5254 return emitOpError() << "applyee CLI has no generator";
5255 auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());
5256 if (!loop)
5257 return emitOpError()
5258 << "currently only supports omp.canonical_loop as applyee";
5259 }
5260 return success();
5261}
5262std::pair<unsigned, unsigned> FuseOp::getApplyeesODSOperandIndexAndLength() {
5263 return getODSOperandIndexAndLength(odsIndex_applyees);
5264}
5265
5266std::pair<unsigned, unsigned> FuseOp::getGenerateesODSOperandIndexAndLength() {
5267 return getODSOperandIndexAndLength(odsIndex_generatees);
5268}
5269
5270//===----------------------------------------------------------------------===//
5271// Critical construct (2.17.1)
5272//===----------------------------------------------------------------------===//
5273
5274void CriticalDeclareOp::build(OpBuilder &builder, OperationState &state,
5275 const CriticalDeclareOperands &clauses) {
5276 CriticalDeclareOp::build(builder, state, clauses.symName,
5277 clauses.symVisibility, clauses.hint);
5278}
5279
5280LogicalResult CriticalDeclareOp::verify() {
5281 return verifySynchronizationHint(*this, getHint());
5282}
5283
5284LogicalResult CriticalOp::verify() {
5285 SymbolRefAttr currentName = getNameAttr();
5286
5287 CriticalOp parentCritical = (*this)->getParentOfType<CriticalOp>();
5288
5289 while (parentCritical) {
5290 SymbolRefAttr parentName = parentCritical.getNameAttr();
5291
5292 if (currentName == parentName) {
5293 if (currentName) {
5294 return emitOpError() << "cannot be nested inside another omp.critical "
5295 "region with the same name ("
5296 << currentName << ")";
5297 } else {
5298 return emitOpError() << "cannot be nested inside another unnamed "
5299 "omp.critical region";
5300 }
5301 }
5302
5303 parentCritical = parentCritical->getParentOfType<CriticalOp>();
5304 }
5305
5306 return success();
5307}
5308
5309LogicalResult CriticalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
5310 if (getNameAttr()) {
5311 SymbolRefAttr symbolRef = getNameAttr();
5312 auto decl = symbolTable.lookupNearestSymbolFrom<CriticalDeclareOp>(
5313 *this, symbolRef);
5314 if (!decl) {
5315 return emitOpError() << "expected symbol reference " << symbolRef
5316 << " to point to a critical declaration";
5317 }
5318 }
5319
5320 return success();
5321}
5322
5323//===----------------------------------------------------------------------===//
5324// Spec 5.1: Error directive (2.5.4)
5325//===----------------------------------------------------------------------===//
5326
5327LogicalResult ErrorOp::verify() {
5328 if (getMessage() && getMessageExpr())
5329 return emitOpError() << "the message must be provided either as a constant "
5330 "`message` attribute or as a `message_expr` "
5331 "operand, but not both";
5332 return success();
5333}
5334
5335//===----------------------------------------------------------------------===//
5336// Ordered construct
5337//===----------------------------------------------------------------------===//
5338
5339static LogicalResult verifyOrderedParent(Operation &op) {
5340 bool hasRegion = op.getNumRegions() > 0;
5341 auto loopOp = op.getParentOfType<LoopNestOp>();
5342 if (!loopOp) {
5343 if (hasRegion)
5344 return success();
5345
5346 // TODO: Consider if this needs to be the case only for the standalone
5347 // variant of the ordered construct.
5348 return op.emitOpError() << "must be nested inside of a loop";
5349 }
5350
5351 Operation *wrapper = loopOp->getParentOp();
5352 if (auto wsloopOp = dyn_cast<WsloopOp>(wrapper)) {
5353 IntegerAttr orderedAttr = wsloopOp.getOrderedAttr();
5354 if (!orderedAttr)
5355 return op.emitOpError() << "the enclosing worksharing-loop region must "
5356 "have an ordered clause";
5357
5358 if (hasRegion && orderedAttr.getInt() != 0)
5359 return op.emitOpError() << "the enclosing loop's ordered clause must not "
5360 "have a parameter present";
5361
5362 if (!hasRegion && orderedAttr.getInt() == 0)
5363 return op.emitOpError() << "the enclosing loop's ordered clause must "
5364 "have a parameter present";
5365 } else if (!isa<SimdOp>(wrapper)) {
5366 return op.emitOpError() << "must be nested inside of a worksharing, simd "
5367 "or worksharing simd loop";
5368 }
5369 return success();
5370}
5371
5372void OrderedOp::build(OpBuilder &builder, OperationState &state,
5373 const OrderedOperands &clauses) {
5374 OrderedOp::build(builder, state, clauses.doacrossDependType,
5375 clauses.doacrossNumLoops, clauses.doacrossDependVars);
5376}
5377
5378LogicalResult OrderedOp::verify() {
5379 if (failed(verifyOrderedParent(**this)))
5380 return failure();
5381
5382 auto wrapper = (*this)->getParentOfType<WsloopOp>();
5383 if (!wrapper || *wrapper.getOrdered() != *getDoacrossNumLoops())
5384 return emitOpError() << "number of variables in depend clause does not "
5385 << "match number of iteration variables in the "
5386 << "doacross loop";
5387
5388 return success();
5389}
5390
5391void OrderedRegionOp::build(OpBuilder &builder, OperationState &state,
5392 const OrderedRegionOperands &clauses) {
5393 OrderedRegionOp::build(builder, state, clauses.parLevelSimd);
5394}
5395
5396LogicalResult OrderedRegionOp::verify() { return verifyOrderedParent(**this); }
5397
5398//===----------------------------------------------------------------------===//
5399// TaskwaitOp
5400//===----------------------------------------------------------------------===//
5401
5402void TaskwaitOp::build(OpBuilder &builder, OperationState &state,
5403 const TaskwaitOperands &clauses) {
5404 // TODO Store clauses in op: depend_iterated_kinds, depend_iterated, nowait.
5405 MLIRContext *ctx = builder.getContext();
5406 TaskwaitOp::build(
5407 builder, state,
5408 /*depend_kinds=*/makeArrayAttr(ctx, clauses.dependKinds),
5409 /*depend_vars=*/clauses.dependVars,
5410 /*depend_iterated_kinds=*/makeArrayAttr(ctx, clauses.dependIteratedKinds),
5411 /*depend_iterated=*/ValueRange(clauses.dependIterated),
5412 /*nowait=*/clauses.nowait);
5413}
5414
5415//===----------------------------------------------------------------------===//
5416// Verifier for AtomicReadOp
5417//===----------------------------------------------------------------------===//
5418
5419LogicalResult AtomicReadOp::verify() {
5420 if (verifyCommon().failed())
5421 return mlir::failure();
5422
5423 int64_t version = 50;
5424 if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5425 if (Attribute verAttr = moduleOp->getDiscardableAttr("omp.version"))
5426 version = llvm::cast<VersionAttr>(verAttr).getVersion();
5427
5428 if (auto mo = getMemoryOrder()) {
5429 if (*mo == ClauseMemoryOrderKind::Release) {
5430 return emitError("memory-order must not be release for atomic reads");
5431 }
5432 if (*mo == ClauseMemoryOrderKind::Acq_rel) {
5433 // acq_rel is prohibited on read only in OpenMP 5.0; allowed in 5.1+.
5434 if (version < 51)
5435 return emitError("memory-order must not be acq_rel for atomic reads");
5436 }
5437 }
5438 return verifySynchronizationHint(*this, getHint());
5439}
5440
5441//===----------------------------------------------------------------------===//
5442// Verifier for AtomicWriteOp
5443//===----------------------------------------------------------------------===//
5444
5445LogicalResult AtomicWriteOp::verify() {
5446 if (verifyCommon().failed())
5447 return mlir::failure();
5448
5449 int64_t version = 50;
5450 if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5451 if (Attribute verAttr = moduleOp->getDiscardableAttr("omp.version"))
5452 version = llvm::cast<VersionAttr>(verAttr).getVersion();
5453
5454 if (auto mo = getMemoryOrder()) {
5455 if (*mo == ClauseMemoryOrderKind::Acquire) {
5456 return emitError("memory-order must not be acquire for atomic writes");
5457 }
5458 if (*mo == ClauseMemoryOrderKind::Acq_rel) {
5459 // acq_rel is prohibited on write only in OpenMP 5.0; allowed in 5.1+.
5460 if (version < 51)
5461 return emitError("memory-order must not be acq_rel for atomic writes");
5462 }
5463 }
5464 return verifySynchronizationHint(*this, getHint());
5465}
5466
5467//===----------------------------------------------------------------------===//
5468// Verifier for AtomicUpdateOp
5469//===----------------------------------------------------------------------===//
5470
5471LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,
5472 PatternRewriter &rewriter) {
5473 if (op.isNoOp()) {
5474 rewriter.eraseOp(op);
5475 return success();
5476 }
5477 if (Value writeVal = op.getWriteOpVal()) {
5478 rewriter.replaceOpWithNewOp<AtomicWriteOp>(
5479 op, op.getX(), writeVal, op.getHintAttr(), op.getMemoryOrderAttr());
5480 return success();
5481 }
5482 return failure();
5483}
5484
5485LogicalResult AtomicUpdateOp::verify() {
5486 if (verifyCommon().failed())
5487 return mlir::failure();
5488
5489 int64_t version = 50;
5490 if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
5491 if (Attribute verAttr = moduleOp->getDiscardableAttr("omp.version"))
5492 version = llvm::cast<VersionAttr>(verAttr).getVersion();
5493
5494 if (auto mo = getMemoryOrder()) {
5495 if (*mo == ClauseMemoryOrderKind::Acq_rel ||
5496 *mo == ClauseMemoryOrderKind::Acquire) {
5497 // This restriction applies only to OpenMP 5.0; removed in 5.1.
5498 if (version < 51)
5499 return emitError(
5500 "memory-order must not be acq_rel or acquire for atomic updates");
5501 }
5502 }
5503
5504 return verifySynchronizationHint(*this, getHint());
5505}
5506
5507LogicalResult AtomicUpdateOp::verifyRegions() { return verifyRegionsCommon(); }
5508
5509//===----------------------------------------------------------------------===//
5510// Verifier for AtomicCaptureOp
5511//===----------------------------------------------------------------------===//
5512
5513AtomicReadOp AtomicCaptureOp::getAtomicReadOp() {
5514 if (auto op = dyn_cast<AtomicReadOp>(getFirstOp()))
5515 return op;
5516 return dyn_cast<AtomicReadOp>(getSecondOp());
5517}
5518
5519AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() {
5520 if (auto op = dyn_cast<AtomicWriteOp>(getFirstOp()))
5521 return op;
5522 return dyn_cast<AtomicWriteOp>(getSecondOp());
5523}
5524
5525AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() {
5526 if (auto op = dyn_cast<AtomicUpdateOp>(getFirstOp()))
5527 return op;
5528 return dyn_cast<AtomicUpdateOp>(getSecondOp());
5529}
5530
5531AtomicCompareOp AtomicCaptureOp::getAtomicCompareOp() {
5532 if (auto op = dyn_cast<AtomicCompareOp>(getFirstOp()))
5533 return op;
5534 return dyn_cast<AtomicCompareOp>(getSecondOp());
5535}
5536
5537LogicalResult AtomicCaptureOp::verify() {
5538 return verifySynchronizationHint(*this, getHint());
5539}
5540
5541LogicalResult AtomicCaptureOp::verifyRegions() {
5542 if (verifyRegionsCommon().failed())
5543 return mlir::failure();
5544
5545 if (getFirstOp()->getInherentAttr("hint").value_or(Attribute{}) ||
5546 getSecondOp()->getInherentAttr("hint").value_or(Attribute{}))
5547 return emitOpError(
5548 "operations inside capture region must not have hint clause");
5549
5550 if (getFirstOp()->getInherentAttr("memory_order").value_or(Attribute{}) ||
5551 getSecondOp()->getInherentAttr("memory_order").value_or(Attribute{}))
5552 return emitOpError(
5553 "operations inside capture region must not have memory_order clause");
5554 return success();
5555}
5556
5557//===----------------------------------------------------------------------===//
5558// AtomicCompareOp
5559//===----------------------------------------------------------------------===//
5560
5561LogicalResult AtomicCompareOp::verify() {
5562 if (verifyCommon().failed())
5563 return mlir::failure();
5564 // OpenMP 5.2 [15.8.3]: the fail clause argument must be one of seq_cst,
5565 // acquire or relaxed ('release' and 'acq_rel' are not valid failure
5566 // orderings and map to invalid cmpxchg failure orderings).
5567 if (auto failOrder = getFailMemoryOrder()) {
5568 if (*failOrder != ClauseMemoryOrderKind::Seq_cst &&
5569 *failOrder != ClauseMemoryOrderKind::Acquire &&
5570 *failOrder != ClauseMemoryOrderKind::Relaxed)
5571 return emitOpError(
5572 "fail_memory_order must be 'seq_cst', 'acquire' or 'relaxed'");
5573 }
5574 return verifySynchronizationHint(*this, getHint());
5575}
5576
5577LogicalResult AtomicCompareOp::verifyRegions() {
5578 if (verifyRegionsCommon().failed())
5579 return mlir::failure();
5580
5581 if (verifyOperator().failed())
5582 return mlir::failure();
5583
5584 Block &block = getRegion().front();
5585
5586 Operation *terminator = block.getTerminator();
5587 if (!terminator || !isa<YieldOp>(terminator))
5588 return emitOpError("region must be terminated with omp.yield");
5589
5590 return success();
5591}
5592
5593//===----------------------------------------------------------------------===//
5594// CancelOp
5595//===----------------------------------------------------------------------===//
5596
5597void CancelOp::build(OpBuilder &builder, OperationState &state,
5598 const CancelOperands &clauses) {
5599 CancelOp::build(builder, state, clauses.cancelDirective, clauses.ifExpr);
5600}
5601
5603 Operation *parent = thisOp->getParentOp();
5604 while (parent) {
5605 if (parent->getDialect() == thisOp->getDialect())
5606 return parent;
5607 parent = parent->getParentOp();
5608 }
5609 return nullptr;
5610}
5611
5612LogicalResult CancelOp::verify() {
5613 ClauseCancellationConstructType cct = getCancelDirective();
5614 // The next OpenMP operation in the chain of parents
5615 Operation *structuralParent = getParentInSameDialect((*this).getOperation());
5616 if (!structuralParent)
5617 return emitOpError() << "Orphaned cancel construct";
5618
5619 if ((cct == ClauseCancellationConstructType::Parallel) &&
5620 !mlir::isa<ParallelOp>(structuralParent)) {
5621 return emitOpError() << "cancel parallel must appear "
5622 << "inside a parallel region";
5623 }
5624 if (cct == ClauseCancellationConstructType::Loop) {
5625 // structural parent will be omp.loop_nest, directly nested inside
5626 // omp.wsloop
5627 auto wsloopOp = mlir::dyn_cast<WsloopOp>(structuralParent->getParentOp());
5628
5629 if (!wsloopOp) {
5630 return emitOpError()
5631 << "cancel loop must appear inside a worksharing-loop region";
5632 }
5633 if (wsloopOp.getNowaitAttr()) {
5634 return emitError() << "A worksharing construct that is canceled "
5635 << "must not have a nowait clause";
5636 }
5637 if (wsloopOp.getOrderedAttr()) {
5638 return emitError() << "A worksharing construct that is canceled "
5639 << "must not have an ordered clause";
5640 }
5641
5642 } else if (cct == ClauseCancellationConstructType::Sections) {
5643 // structural parent will be an omp.section, directly nested inside
5644 // omp.sections
5645 auto sectionsOp =
5646 mlir::dyn_cast<SectionsOp>(structuralParent->getParentOp());
5647 if (!sectionsOp) {
5648 return emitOpError() << "cancel sections must appear "
5649 << "inside a sections region";
5650 }
5651 if (sectionsOp.getNowait()) {
5652 return emitError() << "A sections construct that is canceled "
5653 << "must not have a nowait clause";
5654 }
5655 }
5656 if ((cct == ClauseCancellationConstructType::Taskgroup) &&
5657 (!mlir::isa<omp::TaskOp>(structuralParent) &&
5658 !mlir::isa<omp::TaskloopWrapperOp>(structuralParent->getParentOp()))) {
5659 return emitOpError() << "cancel taskgroup must appear "
5660 << "inside a task region";
5661 }
5662 return success();
5663}
5664
5665//===----------------------------------------------------------------------===//
5666// CancellationPointOp
5667//===----------------------------------------------------------------------===//
5668
5669void CancellationPointOp::build(OpBuilder &builder, OperationState &state,
5670 const CancellationPointOperands &clauses) {
5671 CancellationPointOp::build(builder, state, clauses.cancelDirective);
5672}
5673
5674LogicalResult CancellationPointOp::verify() {
5675 ClauseCancellationConstructType cct = getCancelDirective();
5676 // The next OpenMP operation in the chain of parents
5677 Operation *structuralParent = getParentInSameDialect((*this).getOperation());
5678 if (!structuralParent)
5679 return emitOpError() << "Orphaned cancellation point";
5680
5681 if ((cct == ClauseCancellationConstructType::Parallel) &&
5682 !mlir::isa<ParallelOp>(structuralParent)) {
5683 return emitOpError() << "cancellation point parallel must appear "
5684 << "inside a parallel region";
5685 }
5686 // Strucutal parent here will be an omp.loop_nest. Get the parent of that to
5687 // find the wsloop
5688 if ((cct == ClauseCancellationConstructType::Loop) &&
5689 !mlir::isa<WsloopOp>(structuralParent->getParentOp())) {
5690 return emitOpError() << "cancellation point loop must appear "
5691 << "inside a worksharing-loop region";
5692 }
5693 if ((cct == ClauseCancellationConstructType::Sections) &&
5694 !mlir::isa<omp::SectionOp>(structuralParent)) {
5695 return emitOpError() << "cancellation point sections must appear "
5696 << "inside a sections region";
5697 }
5698 if ((cct == ClauseCancellationConstructType::Taskgroup) &&
5699 (!mlir::isa<omp::TaskOp>(structuralParent) &&
5700 !mlir::isa<omp::TaskloopWrapperOp>(structuralParent->getParentOp()))) {
5701 return emitOpError() << "cancellation point taskgroup must appear "
5702 << "inside a task region";
5703 }
5704 return success();
5705}
5706
5707//===----------------------------------------------------------------------===//
5708// MapBoundsOp
5709//===----------------------------------------------------------------------===//
5710
5711LogicalResult MapBoundsOp::verify() {
5712 auto extent = getExtent();
5713 auto upperbound = getUpperBound();
5714 if (!extent && !upperbound)
5715 return emitError("expected extent or upperbound.");
5716 return success();
5717}
5718
5719void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,
5720 TypeRange /*result_types*/, StringAttr symName,
5721 TypeAttr type) {
5722 PrivateClauseOp::build(
5723 odsBuilder, odsState, symName, /*sym_visibility=*/nullptr, type,
5724 DataSharingClauseTypeAttr::get(odsBuilder.getContext(),
5725 DataSharingClauseType::Private));
5726}
5727
5728LogicalResult PrivateClauseOp::verifyRegions() {
5729 Type argType = getArgType();
5730 auto verifyTerminator = [&](Operation *terminator, bool yieldsValue,
5731 bool yieldsAnyType) -> LogicalResult {
5732 if (!terminator->getBlock()->getSuccessors().empty())
5733 return success();
5734
5735 if (!llvm::isa<YieldOp>(terminator))
5736 return mlir::emitError(terminator->getLoc())
5737 << "expected exit block terminator to be an `omp.yield` op.";
5738
5739 YieldOp yieldOp = llvm::cast<YieldOp>(terminator);
5740 TypeRange yieldedTypes = yieldOp.getResults().getTypes();
5741
5742 if (!yieldsValue) {
5743 if (yieldedTypes.empty())
5744 return success();
5745
5746 return mlir::emitError(terminator->getLoc())
5747 << "Did not expect any values to be yielded.";
5748 }
5749
5750 if (yieldedTypes.size() == 1 &&
5751 (yieldsAnyType || yieldedTypes.front() == argType))
5752 return success();
5753
5754 auto error = mlir::emitError(yieldOp.getLoc())
5755 << "Invalid yielded value. Expected ";
5756 if (yieldsAnyType)
5757 error << "a single value";
5758 else
5759 error << "type: " << argType;
5760 error << ", got: ";
5761
5762 if (yieldedTypes.empty())
5763 error << "None";
5764 else
5765 error << yieldedTypes;
5766
5767 return error;
5768 };
5769
5770 auto verifyRegion = [&](Region &region, unsigned expectedNumArgs,
5771 StringRef regionName, bool yieldsValue,
5772 bool yieldsAnyType = false) -> LogicalResult {
5773 assert(!region.empty());
5774
5775 if (region.getNumArguments() != expectedNumArgs)
5776 return mlir::emitError(region.getLoc())
5777 << "`" << regionName << "`: " << "expected " << expectedNumArgs
5778 << " region arguments, got: " << region.getNumArguments();
5779
5780 for (Block &block : region) {
5781 // MLIR will verify the absence of the terminator for us.
5782 if (!block.mightHaveTerminator())
5783 continue;
5784
5785 if (failed(verifyTerminator(block.getTerminator(), yieldsValue,
5786 yieldsAnyType)))
5787 return failure();
5788 }
5789
5790 return success();
5791 };
5792
5793 // Ensure all of the region arguments have the same type
5794 for (Region *region : getRegions())
5795 for (Type ty : region->getArgumentTypes())
5796 if (ty != argType)
5797 return emitError() << "Region argument type mismatch: got " << ty
5798 << " expected " << argType << ".";
5799
5800 mlir::Region &initRegion = getInitRegion();
5801 if (!initRegion.empty() &&
5802 failed(verifyRegion(getInitRegion(), /*expectedNumArgs=*/2, "init",
5803 /*yieldsValue=*/true)))
5804 return failure();
5805
5806 DataSharingClauseType dsType = getDataSharingType();
5807
5808 if (dsType == DataSharingClauseType::Private && !getCopyRegion().empty())
5809 return emitError("`private` clauses do not require a `copy` region.");
5810
5811 if (dsType == DataSharingClauseType::FirstPrivate && getCopyRegion().empty())
5812 return emitError(
5813 "`firstprivate` clauses require at least a `copy` region.");
5814
5815 if (dsType == DataSharingClauseType::FirstPrivate &&
5816 failed(verifyRegion(getCopyRegion(), /*expectedNumArgs=*/2, "copy",
5817 /*yieldsValue=*/true)))
5818 return failure();
5819
5820 if (dsType == DataSharingClauseType::Linear && !getCopyRegion().empty())
5821 return emitError("`linear` clauses do not require a `copy` region.");
5822
5823 if (!getDataAddressRegion().empty()) {
5824 if (dsType != DataSharingClauseType::Linear)
5825 return emitError(
5826 "only `linear` clauses can have a `data_address` region.");
5827 if (failed(verifyRegion(getDataAddressRegion(), /*expectedNumArgs=*/1,
5828 "data_address", /*yieldsValue=*/true,
5829 /*yieldsAnyType=*/true)))
5830 return failure();
5831 }
5832
5833 if (!getDeallocRegion().empty() &&
5834 failed(verifyRegion(getDeallocRegion(), /*expectedNumArgs=*/1, "dealloc",
5835 /*yieldsValue=*/false)))
5836 return failure();
5837
5838 return success();
5839}
5840
5841//===----------------------------------------------------------------------===//
5842// Spec 5.2: Masked construct (10.5)
5843//===----------------------------------------------------------------------===//
5844
5845void MaskedOp::build(OpBuilder &builder, OperationState &state,
5846 const MaskedOperands &clauses) {
5847 MaskedOp::build(builder, state, clauses.filteredThreadId);
5848}
5849
5850//===----------------------------------------------------------------------===//
5851// Spec 5.2: Dispatch construct (7.6)
5852//===----------------------------------------------------------------------===//
5853
5854void DispatchOp::build(OpBuilder &builder, OperationState &state,
5855 const DispatchOperands &clauses) {
5856 DispatchOp::build(builder, state, clauses.nocontext, clauses.novariants,
5857 clauses.nowait);
5858}
5859
5860//===----------------------------------------------------------------------===//
5861// Spec 5.2: Scan construct (5.6)
5862//===----------------------------------------------------------------------===//
5863
5864void ScanOp::build(OpBuilder &builder, OperationState &state,
5865 const ScanOperands &clauses) {
5866 ScanOp::build(builder, state, clauses.inclusiveVars, clauses.exclusiveVars);
5867}
5868
5869LogicalResult ScanOp::verify() {
5870 if (hasExclusiveVars() == hasInclusiveVars())
5871 return emitError(
5872 "Exactly one of EXCLUSIVE or INCLUSIVE clause is expected");
5873 if (WsloopOp parentWsLoopOp = (*this)->getParentOfType<WsloopOp>()) {
5874 if (parentWsLoopOp.hasReductionModifier(ReductionModifier::inscan))
5875 return success();
5876 }
5877 if (SimdOp parentSimdOp = (*this)->getParentOfType<SimdOp>()) {
5878 if (parentSimdOp.hasReductionModifier(ReductionModifier::inscan))
5879 return success();
5880 }
5881 return emitError("SCAN directive needs to be enclosed within a parent "
5882 "worksharing loop construct or SIMD construct with INSCAN "
5883 "reduction modifier");
5884}
5885
5886/// Verifies align clause in allocate directive
5887LogicalResult verifyAlignment(Operation &op,
5888 std::optional<uint64_t> alignment) {
5889 if (alignment.has_value()) {
5890 if ((alignment.value() != 0) && !llvm::has_single_bit(alignment.value()))
5891 return op.emitError()
5892 << "ALIGN value : " << alignment.value() << " must be power of 2";
5893 }
5894 return success();
5895}
5896
5897LogicalResult AllocateDirOp::verify() {
5898 return verifyAlignment(*getOperation(), getAlign());
5899}
5900
5901//===----------------------------------------------------------------------===//
5902// AllocSharedMemOp
5903//===----------------------------------------------------------------------===//
5904
5905LogicalResult AllocSharedMemOp::verify() {
5906 return verifyAlignment(*getOperation(), getMemAlignment());
5907}
5908
5909//===----------------------------------------------------------------------===//
5910// FreeSharedMemOp
5911//===----------------------------------------------------------------------===//
5912
5913LogicalResult FreeSharedMemOp::verify() {
5914 return verifyAlignment(*getOperation(), getMemAlignment());
5915}
5916
5917//===----------------------------------------------------------------------===//
5918// WorkdistributeOp
5919//===----------------------------------------------------------------------===//
5920
5921LogicalResult WorkdistributeOp::verify() {
5922 if (isCombined())
5923 return emitOpError() << "cannot be a non-innermost combined construct leaf";
5924
5925 // Check that region exists and is not empty
5926 Region &region = getRegion();
5927 if (region.empty())
5928 return emitOpError("region cannot be empty");
5929 // Verify single entry point.
5930 Block &entryBlock = region.front();
5931 if (entryBlock.empty())
5932 return emitOpError("region must contain a structured block");
5933 // Verify single exit point.
5934 bool hasTerminator = false;
5935 for (Block &block : region) {
5936 if (isa<TerminatorOp>(block.back())) {
5937 if (hasTerminator) {
5938 return emitOpError("region must have exactly one terminator");
5939 }
5940 hasTerminator = true;
5941 }
5942 }
5943 if (!hasTerminator) {
5944 return emitOpError("region must be terminated with omp.terminator");
5945 }
5946 auto walkResult = region.walk([&](Operation *op) -> WalkResult {
5947 // No implicit barrier at end
5948 if (isa<BarrierOp>(op)) {
5949 return emitOpError(
5950 "explicit barriers are not allowed in workdistribute region");
5951 }
5952 // Check for invalid nested constructs
5953 if (isa<ParallelOp>(op)) {
5954 return emitOpError(
5955 "nested parallel constructs not allowed in workdistribute");
5956 }
5957 if (isa<TeamsOp>(op)) {
5958 return emitOpError(
5959 "nested teams constructs not allowed in workdistribute");
5960 }
5961 return WalkResult::advance();
5962 });
5963 if (walkResult.wasInterrupted())
5964 return failure();
5965
5966 Operation *parentOp = (*this)->getParentOp();
5967 if (!llvm::dyn_cast<TeamsOp>(parentOp))
5968 return emitOpError("workdistribute must be nested under teams");
5969 return success();
5970}
5971
5972//===----------------------------------------------------------------------===//
5973// Declare simd [7.7]
5974//===----------------------------------------------------------------------===//
5975
5976LogicalResult DeclareSimdOp::verify() {
5977 // Must be nested inside a function-like op
5978 auto func =
5979 dyn_cast_if_present<mlir::FunctionOpInterface>((*this)->getParentOp());
5980 if (!func)
5981 return emitOpError() << "must be nested inside a function";
5982
5983 if (getInbranch() && getNotinbranch())
5984 return emitOpError("cannot have both 'inbranch' and 'notinbranch'");
5985
5986 if (failed(verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars(),
5987 /*isDeclareSimd=*/true)))
5988 return failure();
5989
5990 if (getLinearSyms())
5991 return emitOpError("linear privatizers are not allowed");
5992
5993 return verifyAlignedClause(*this, getAlignments(), getAlignedVars());
5994}
5995
5996void DeclareSimdOp::build(OpBuilder &odsBuilder, OperationState &odsState,
5997 const DeclareSimdOperands &clauses) {
5998 MLIRContext *ctx = odsBuilder.getContext();
5999 DeclareSimdOp::build(
6000 odsBuilder, odsState, clauses.alignedVars,
6001 makeArrayAttr(ctx, clauses.alignments), clauses.inbranch,
6002 clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
6003 clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
6004 clauses.notinbranch, clauses.simdlen, clauses.uniformVars);
6005}
6006
6007//===----------------------------------------------------------------------===//
6008// Parser and printer for Uniform Clause
6009//===----------------------------------------------------------------------===//
6010
6011/// uniform ::= `uniform` `(` uniform-list `)`
6012/// uniform-list := uniform-val (`,` uniform-val)*
6013/// uniform-val := ssa-id `:` type
6014static ParseResult
6017 SmallVectorImpl<Type> &uniformTypes) {
6018 return parser.parseCommaSeparatedList([&]() -> mlir::ParseResult {
6019 if (parser.parseOperand(uniformVars.emplace_back()) ||
6020 parser.parseColonType(uniformTypes.emplace_back()))
6021 return mlir::failure();
6022 return mlir::success();
6023 });
6024}
6025
6026/// Print Uniform Clauses
6028 ValueRange uniformVars, TypeRange uniformTypes) {
6029 for (unsigned i = 0; i < uniformVars.size(); ++i) {
6030 if (i != 0)
6031 p << ", ";
6032 p << uniformVars[i] << " : " << uniformTypes[i];
6033 }
6034}
6035
6036//===----------------------------------------------------------------------===//
6037// Parser and printer for Affinity Clause
6038//===----------------------------------------------------------------------===//
6039
6040static ParseResult parseAffinityClause(
6041 OpAsmParser &parser,
6044 SmallVectorImpl<Type> &iteratedTypes,
6045 SmallVectorImpl<Type> &affinityVarTypes) {
6046 if (failed(parseSplitIteratedList(
6047 parser, iterated, iteratedTypes, affinityVars, affinityVarTypes,
6048 /*parsePrefix=*/[&]() -> ParseResult { return success(); })))
6049 return failure();
6050 return success();
6051}
6052
6054 ValueRange iterated, ValueRange affinityVars,
6055 TypeRange iteratedTypes,
6056 TypeRange affinityVarTypes) {
6057 auto nop = [&](Value, Type) {};
6058 printSplitIteratedList(p, iterated, iteratedTypes, affinityVars,
6059 affinityVarTypes,
6060 /*plain prefix*/ nop,
6061 /*iterated prefix*/ nop);
6062}
6063
6064//===----------------------------------------------------------------------===//
6065// Parser, printer, and verifier for Iterator modifier
6066//===----------------------------------------------------------------------===//
6067
6068static ParseResult
6073 SmallVectorImpl<Type> &lbTypes,
6074 SmallVectorImpl<Type> &ubTypes,
6075 SmallVectorImpl<Type> &stepTypes) {
6076
6077 llvm::SMLoc ivLoc = parser.getCurrentLocation();
6079
6080 // Parse induction variables: %i : i32, %j : i32
6081 if (parser.parseCommaSeparatedList([&]() -> ParseResult {
6082 OpAsmParser::Argument &arg = ivArgs.emplace_back();
6083 if (parser.parseArgument(arg))
6084 return failure();
6085
6086 // Optional type, default to Index if not provided
6087 if (succeeded(parser.parseOptionalColon())) {
6088 if (parser.parseType(arg.type))
6089 return failure();
6090 } else {
6091 arg.type = parser.getBuilder().getIndexType();
6092 }
6093 return success();
6094 }))
6095 return failure();
6096
6097 // ) = (
6098 if (parser.parseRParen() || parser.parseEqual() || parser.parseLParen())
6099 return failure();
6100
6101 // Parse Ranges: (%lb to %ub step %st, ...)
6102 if (parser.parseCommaSeparatedList([&]() -> ParseResult {
6103 OpAsmParser::UnresolvedOperand lb, ub, st;
6104 if (parser.parseOperand(lb) || parser.parseKeyword("to") ||
6105 parser.parseOperand(ub) || parser.parseKeyword("step") ||
6106 parser.parseOperand(st))
6107 return failure();
6108
6109 lbs.push_back(lb);
6110 ubs.push_back(ub);
6111 steps.push_back(st);
6112 return success();
6113 }))
6114 return failure();
6115
6116 if (parser.parseRParen())
6117 return failure();
6118
6119 if (ivArgs.size() != lbs.size())
6120 return parser.emitError(ivLoc)
6121 << "mismatch: " << ivArgs.size() << " variables but " << lbs.size()
6122 << " ranges";
6123
6124 for (auto &arg : ivArgs) {
6125 lbTypes.push_back(arg.type);
6126 ubTypes.push_back(arg.type);
6127 stepTypes.push_back(arg.type);
6128 }
6129
6130 return parser.parseRegion(region, ivArgs);
6131}
6132
6134 ValueRange lbs, ValueRange ubs,
6136 TypeRange) {
6137 Block &entry = region.front();
6138
6139 for (unsigned i = 0, e = entry.getNumArguments(); i < e; ++i) {
6140 if (i != 0)
6141 p << ", ";
6142 p.printRegionArgument(entry.getArgument(i));
6143 }
6144 p << ") = (";
6145
6146 // (%lb0 to %ub0 step %step0, %lb1 to %ub1 step %step1, ...)
6147 for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
6148 if (i)
6149 p << ", ";
6150 p << lbs[i] << " to " << ubs[i] << " step " << steps[i];
6151 }
6152 p << ") ";
6153
6154 p.printRegion(region, /*printEntryBlockArgs=*/false,
6155 /*printBlockTerminators=*/true);
6156}
6157
6158YieldOp IteratorOp::getYieldOp() {
6159 YieldOp yield;
6160 for (Block &block : getRegion()) {
6161 auto blockYield =
6162 dyn_cast_if_present<YieldOp>(block.empty() ? nullptr : &block.back());
6163 if (!blockYield)
6164 continue;
6165 if (yield)
6166 return {};
6167 yield = blockYield;
6168 }
6169 return yield;
6170}
6171
6172LogicalResult IteratorOp::verify() {
6173 auto iteratedTy = llvm::dyn_cast<omp::IteratedType>(getIterated().getType());
6174 if (!iteratedTy)
6175 return emitOpError() << "result must be omp.iterated<entry_ty>";
6176
6177 Block::BlockArgListType ivs = getRegion().front().getArguments();
6178 if (getLoopLowerBounds().size() != ivs.size())
6179 return emitOpError() << "number of range arguments and IVs do not match";
6180
6181 for (auto [lb, iv] : llvm::zip_equal(getLoopLowerBounds(), ivs)) {
6182 if (lb.getType() != iv.getType())
6183 return emitOpError()
6184 << "range argument type does not match corresponding IV type";
6185 }
6186
6187 for (Value step : getLoopSteps()) {
6188 if (matchPattern(step, m_Zero()))
6189 return emitOpError() << "loop step must not be zero";
6190 }
6191
6192 YieldOp yield = getYieldOp();
6193 if (!yield) {
6194 bool hasYield = llvm::any_of(getRegion(), [](Block &block) {
6195 return !block.empty() && isa<YieldOp>(block.back());
6196 });
6197 if (hasYield)
6198 return emitOpError() << "region must contain exactly one omp.yield";
6199 return emitOpError() << "region must be terminated by omp.yield";
6200 }
6201
6202 if (yield.getNumOperands() != 1)
6203 return emitOpError()
6204 << "omp.yield in omp.iterator region must yield exactly one value";
6205
6206 mlir::Type yieldedTy = yield.getOperand(0).getType();
6207 mlir::Type elemTy = iteratedTy.getElementType();
6208
6209 if (yieldedTy != elemTy)
6210 return emitOpError() << "omp.iterated element type (" << elemTy
6211 << ") does not match omp.yield operand type ("
6212 << yieldedTy << ")";
6213
6214 return success();
6215}
6216
6217LogicalResult IteratorOp::verifyRegions() {
6218 // Translation routes only the omp.yield block back to the iterator loop.
6219 for (Block &block : getRegion()) {
6220 Operation *terminator = block.getTerminator();
6221 if (block.getNumSuccessors() == 0 && !isa<YieldOp>(terminator))
6222 return mlir::emitError(terminator->getLoc())
6223 << "expected exit block terminator to be an `omp.yield` op.";
6224 }
6225 return success();
6226}
6227
6228//===----------------------------------------------------------------------===//
6229// GroupprivateOp
6230//===----------------------------------------------------------------------===//
6231
6232LogicalResult
6233GroupprivateOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
6234 auto *symbol = symbolTable.lookupNearestSymbolFrom(*this, getSymNameAttr());
6235 if (!symbol)
6236 return emitOpError() << "expected symbol reference '" << getSymName()
6237 << "' to point to a global variable";
6238
6239 if (isa<FunctionOpInterface>(symbol))
6240 return emitOpError() << "expected symbol reference '" << getSymName()
6241 << "' to point to a global variable, not a function";
6242
6243 return success();
6244}
6245
6246#define GET_ATTRDEF_CLASSES
6247#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
6248
6249#define GET_OP_CLASSES
6250#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
6251
6252#define GET_TYPEDEF_CLASSES
6253#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
return success()
if(failed(verifyVectorMemoryOp(getOperation(), memrefType, getVectorType()))) return failure()
static std::optional< int64_t > getUpperBound(Value iv)
Gets the constant upper bound on an affine.for iv.
static LogicalResult verifyRegion(emitc::SwitchOp op, Region &region, const Twine &name)
Definition EmitC.cpp:1521
b
Return true if permutation is a valid permutation of the outer_dims_perm (case OuterOrInnerPerm::Oute...
ArrayAttr()
b getContext())
static const mlir::GenInfo * generator
static LogicalResult verifyNontemporalClause(Operation *op, OperandRange nontemporalVars)
static DenseI64ArrayAttr makeDenseI64ArrayAttr(MLIRContext *ctx, const ArrayRef< int64_t > intArray)
static void printDependVarList(OpAsmPrinter &p, Operation *op, OperandRange dependVars, TypeRange dependTypes, std::optional< ArrayAttr > dependKinds, OperandRange iteratedVars, TypeRange iteratedTypes, std::optional< ArrayAttr > iteratedKinds)
Print Depend clause.
static ParseResult parseTargetOpRegion(OpAsmParser &parser, Region &region, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &hasDeviceAddrVars, SmallVectorImpl< Type > &hasDeviceAddrTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &hostEvalVars, SmallVectorImpl< Type > &hostEvalTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &mapVars, SmallVectorImpl< Type > &mapTypes, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier, DenseI64ArrayAttr &privateMaps)
static constexpr StringRef getPrivateNeedsBarrierSpelling()
static void printHeapAllocClause(OpAsmPrinter &p, Operation *op, TypeAttr inType, ValueRange typeparams, TypeRange typeparamsTypes, ValueRange shape, TypeRange shapeTypes)
static ParseResult parseInReductionPrivateReductionRegion(OpAsmParser &parser, Region &region, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &inReductionVars, SmallVectorImpl< Type > &inReductionTypes, DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &reductionVars, SmallVectorImpl< Type > &reductionTypes, DenseBoolArrayAttr &reductionByref, ArrayAttr &reductionSyms)
static ParseResult parseInReductionPrivateRegion(OpAsmParser &parser, Region &region, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &inReductionVars, SmallVectorImpl< Type > &inReductionTypes, DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier)
static ArrayAttr makeArrayAttr(MLIRContext *context, llvm::ArrayRef< Attribute > attrs)
static ParseResult parseClauseAttr(AsmParser &parser, ClauseAttr &attr)
static void printLinearPrivateReductionRegion(OpAsmPrinter &p, Operation *op, Region &region, ValueRange linearVars, TypeRange linearTypes, ValueRange linearStepVars, TypeRange linearStepTypes, ArrayAttr linearVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier, ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes, DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms)
static ParseResult parseLinearPrivateReductionRegion(OpAsmParser &parser, Region &region, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &linearVars, SmallVectorImpl< Type > &linearTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &linearStepVars, SmallVectorImpl< Type > &linearStepTypes, ArrayAttr &linearVarTypes, ArrayAttr &linearModifiers, ArrayAttr &linearSyms, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &reductionVars, SmallVectorImpl< Type > &reductionTypes, DenseBoolArrayAttr &reductionByref, ArrayAttr &reductionSyms)
static void printDynGroupprivateClause(OpAsmPrinter &printer, Operation *op, AccessGroupModifierAttr modifierFirst, FallbackModifierAttr modifierSecond, Value dynGroupprivateSize, Type sizeType)
static void printAllocateAndAllocator(OpAsmPrinter &p, Operation *op, OperandRange allocateVars, TypeRange allocateTypes, OperandRange allocatorVars, TypeRange allocatorTypes)
Print allocate clause.
static DenseBoolArrayAttr makeDenseBoolArrayAttr(MLIRContext *ctx, const ArrayRef< bool > boolArray)
static std::string generateLoopNestingName(StringRef prefix, CanonicalLoopOp op)
Generate a name of a canonical loop nest of the format <prefix>(_r<idx>_s<idx>)*.
static ParseResult parseAffinityClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &iterated, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &affinityVars, SmallVectorImpl< Type > &iteratedTypes, SmallVectorImpl< Type > &affinityVarTypes)
static void printSplitIteratedList(OpAsmPrinter &p, ValueRange iteratedVars, TypeRange iteratedTypes, ValueRange plainVars, TypeRange plainTypes, PrintPrefixFn &&printPrefixForPlain, PrintPrefixFn &&printPrefixForIterated)
static LogicalResult verifyDependVarList(Operation *op, std::optional< ArrayAttr > dependKinds, OperandRange dependVars, std::optional< ArrayAttr > iteratedKinds, OperandRange iteratedVars)
Verifies Depend clause.
static void addUnknownMemoryEffects(SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Add arbitrary read, write, allocate, and free effects.
static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName, ValueRange argsSubrange, std::optional< MapPrintArgs > mapArgs)
static void printAffinityClause(OpAsmPrinter &p, Operation *op, ValueRange iterated, ValueRange affinityVars, TypeRange iteratedTypes, TypeRange affinityVarTypes)
static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region &region, const AllRegionPrintArgs &args)
static ParseResult parseGranularityClause(OpAsmParser &parser, ClauseTypeAttr &prescriptiveness, std::optional< OpAsmParser::UnresolvedOperand > &operand, Type &operandType, std::optional< ClauseType >(*symbolizeClause)(StringRef), StringRef clauseName)
static void printIteratorHeader(OpAsmPrinter &p, Operation *op, Region &region, ValueRange lbs, ValueRange ubs, ValueRange steps, TypeRange, TypeRange, TypeRange)
static LogicalResult verifyDeclareTargetAttr(Operation *op, Attribute attr)
static ParseResult parseHeapAllocClause(OpAsmParser &parser, TypeAttr &inTypeAttr, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &typeparams, SmallVectorImpl< Type > &typeparamsTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &shape, SmallVectorImpl< Type > &shapeTypes)
operation ::= $in_type ( ( $typeparams ) )? ( , $shape )?
static void printInReductionClause(OpAsmPrinter &p, Operation *op, ValueRange inReductionVars, TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref, ArrayAttr inReductionSyms)
Prints an in_reduction clause for an operation that does not give its list items entry block argument...
static void printClauseWithRegionArgs(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName, ValueRange argsSubrange, ValueRange operands, TypeRange types, ArrayAttr symbols=nullptr, DenseI64ArrayAttr mapIndices=nullptr, DenseBoolArrayAttr byref=nullptr, ArrayAttr modifier=nullptr, UnitAttr needsBarrier=nullptr)
static ParseResult parseIteratorHeader(OpAsmParser &parser, Region &region, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &lbs, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &ubs, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &steps, SmallVectorImpl< Type > &lbTypes, SmallVectorImpl< Type > &ubTypes, SmallVectorImpl< Type > &stepTypes)
static LogicalResult verifyReductionVarList(Operation *op, std::optional< ArrayAttr > reductionSyms, OperandRange reductionVars, std::optional< ArrayRef< bool > > reductionByref, ArrayAttr reductionMods=nullptr)
Verifies Reduction Clause.
static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region &region, AllRegionParseArgs args)
static ParseResult parseLoopTransformClis(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &generateesOperands, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &applyeesOperands)
static ParseResult parseSynchronizationHint(OpAsmParser &parser, IntegerAttr &hintAttr)
Parses a Synchronization Hint clause.
static void printLinearClause(OpAsmPrinter &p, Operation *op, ValueRange linearVars, TypeRange linearTypes, ValueRange linearStepVars, TypeRange stepVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms)
Print Linear Clause.
static void printScheduleClause(OpAsmPrinter &p, Operation *op, ClauseScheduleKindAttr scheduleKind, ScheduleModifierAttr scheduleMod, UnitAttr scheduleSimd, Value scheduleChunk, Type scheduleChunkType)
Print schedule clause.
static ParseResult parsePrivateReductionRegion(OpAsmParser &parser, Region &region, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier, ArrayAttr &reductionMods, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &reductionVars, SmallVectorImpl< Type > &reductionTypes, DenseBoolArrayAttr &reductionByref, ArrayAttr &reductionSyms)
static void printCopyprivate(OpAsmPrinter &p, Operation *op, OperandRange copyprivateVars, TypeRange copyprivateTypes, std::optional< ArrayAttr > copyprivateSyms)
Print Copyprivate clause.
static ParseResult parseOrderClause(OpAsmParser &parser, ClauseOrderKindAttr &order, OrderModifierAttr &orderMod)
static bool mapTypeToBool(ClauseMapFlags value, ClauseMapFlags flag)
static void printAlignedClause(OpAsmPrinter &p, Operation *op, ValueRange alignedVars, TypeRange alignedTypes, std::optional< ArrayAttr > alignments)
Print Aligned Clause.
static bool targetInReductionCapturedBy(Value inReductionVar, Value mapVarPtr)
An omp.target in_reduction operand is captured by a map_entries entry when the entry's MapInfoOp var_...
static LogicalResult verifySynchronizationHint(Operation *op, uint64_t hint)
Verifies a synchronization hint clause.
static ParseResult parseUseDeviceAddrUseDevicePtrRegion(OpAsmParser &parser, Region &region, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &useDeviceAddrVars, SmallVectorImpl< Type > &useDeviceAddrTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &useDevicePtrVars, SmallVectorImpl< Type > &useDevicePtrTypes)
static void printPrivateReductionRegion(OpAsmPrinter &p, Operation *op, Region &region, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier, ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes, DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms)
static ParseResult parseUniformClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &uniformVars, SmallVectorImpl< Type > &uniformTypes)
uniform ::= uniform ( uniform-list ) uniform-list := uniform-val (, uniform-val)* uniform-val := ssa-...
static void printInReductionPrivateRegion(OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars, TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref, ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier)
static LogicalResult verifyAllocateClause(Operation *op, ValueRange allocateVars, ValueRange allocatorVars, DenseI64ArrayAttr allocateAlignments, DenseI64ArrayAttr allocatePrivateIndices, ValueRange privateVars={}, ArrayAttr privateSyms=nullptr, bool requirePrivateIndices=false)
static void printSynchronizationHint(OpAsmPrinter &p, Operation *op, IntegerAttr hintAttr)
Prints a Synchronization Hint clause.
static void printGranularityClause(OpAsmPrinter &p, Operation *op, ClauseTypeAttr prescriptiveness, Value operand, mlir::Type operandType, StringRef(*stringifyClauseType)(ClauseType))
static ParseResult parseDependVarList(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &dependVars, SmallVectorImpl< Type > &dependTypes, ArrayAttr &dependKinds, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &iteratedVars, SmallVectorImpl< Type > &iteratedTypes, ArrayAttr &iteratedKinds)
depend-entry-list ::= depend-entry | depend-entry-list , depend-entry depend-entry ::= depend-kind ->...
static void getTaskReductionEffects(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
static Operation * getParentInSameDialect(Operation *thisOp)
static void printUniformClause(OpAsmPrinter &p, Operation *op, ValueRange uniformVars, TypeRange uniformTypes)
Print Uniform Clauses.
static LogicalResult verifyCopyprivateVarList(Operation *op, OperandRange copyprivateVars, std::optional< ArrayAttr > copyprivateSyms)
Verifies CopyPrivate Clause.
static LogicalResult verifyAlignedClause(Operation *op, std::optional< ArrayAttr > alignments, OperandRange alignedVars)
static ParseResult parsePrivateRegion(OpAsmParser &parser, Region &region, llvm::SmallVectorImpl< OpAsmParser::UnresolvedOperand > &privateVars, llvm::SmallVectorImpl< Type > &privateTypes, ArrayAttr &privateSyms, UnitAttr &privateNeedsBarrier)
static void printNumTasksClause(OpAsmPrinter &p, Operation *op, ClauseNumTasksTypeAttr numTasksMod, Value numTasks, mlir::Type numTasksType)
static void printLoopTransformClis(OpAsmPrinter &p, TileOp op, OperandRange generatees, OperandRange applyees)
static ParseResult parseDynGroupprivateClause(OpAsmParser &parser, AccessGroupModifierAttr &accessGroupAttr, FallbackModifierAttr &fallbackAttr, std::optional< OpAsmParser::UnresolvedOperand > &dynGroupprivateSize, Type &sizeType)
static void printPrivateRegion(OpAsmPrinter &p, Operation *op, Region &region, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier)
static ParseResult parseSplitIteratedList(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &iteratedVars, SmallVectorImpl< Type > &iteratedTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &plainVars, SmallVectorImpl< Type > &plainTypes, ParsePrefixFn &&parsePrefix)
static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op, Region &region, ValueRange taskReductionVars, TypeRange taskReductionTypes, DenseBoolArrayAttr taskReductionByref, ArrayAttr taskReductionSyms)
static LogicalResult verifyMapInfoForMapClause(Operation *op, mlir::omp::MapInfoOp mapInfoOp, llvm::DenseSet< mlir::TypedValue< mlir::omp::PointerLikeType > > &updateToVars, llvm::DenseSet< mlir::TypedValue< mlir::omp::PointerLikeType > > &updateFromVars)
static void getPrivatisationEffects(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Privatisation may read the original as a mold or firstprivate copy source. Any lastprivate copy-back ...
static void printInReductionPrivateReductionRegion(OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars, TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref, ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier, ArrayAttr reductionMods, ValueRange reductionVars, TypeRange reductionTypes, DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms)
return success()
static void getBarrierEffects(llvm::SmallVectorImpl< SideEffects::EffectInstance< MemoryEffects::Effect > > &effects)
static LogicalResult verifyOrderedParent(Operation &op)
static void printOrderClause(OpAsmPrinter &p, Operation *op, ClauseOrderKindAttr order, OrderModifierAttr orderMod)
static ParseResult parseBlockArgClause(OpAsmParser &parser, llvm::SmallVectorImpl< OpAsmParser::Argument > &entryBlockArgs, StringRef keyword, std::optional< MapParseArgs > mapArgs)
static LogicalResult verifyPrivateVarsMapping(TargetOp targetOp)
static ParseResult parseScheduleClause(OpAsmParser &parser, ClauseScheduleKindAttr &scheduleAttr, ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd, std::optional< OpAsmParser::UnresolvedOperand > &chunkSize, Type &chunkType)
schedule ::= schedule ( sched-list ) sched-list ::= sched-val | sched-val sched-list | sched-val ,...
static LogicalResult verifyDynGroupprivateClause(Operation *op, AccessGroupModifierAttr accessGroup, FallbackModifierAttr fallback, Value dynGroupprivateSize)
static LogicalResult verifyLinearModifiers(Operation *op, std::optional< ArrayAttr > linearModifiers, OperandRange linearVars, bool isDeclareSimd=false)
OpenMP 5.2, Section 5.4.6: "A linear-modifier may be specified as ref or uval only on a declare simd ...
static void printClauseAttr(OpAsmPrinter &p, Operation *op, ClauseAttr attr)
static LogicalResult verifyLinearSyms(OpType op)
Verifies that each linear variable has a linear privatizer.
static ParseResult parseAllocateAndAllocator(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &allocateVars, SmallVectorImpl< Type > &allocateTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &allocatorVars, SmallVectorImpl< Type > &allocatorTypes)
Parse an allocate clause with allocators and a list of operands with types.
static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op, ArrayAttr membersIdx)
static void getReductionEffects(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Reductions initialise private accumulators and combine them into the original variables....
static ParseResult parseClauseWithRegionArgs(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &operands, SmallVectorImpl< Type > &types, SmallVectorImpl< OpAsmParser::Argument > &regionPrivateArgs, ArrayAttr *symbols=nullptr, DenseI64ArrayAttr *mapIndices=nullptr, DenseBoolArrayAttr *byref=nullptr, ArrayAttr *modifier=nullptr, UnitAttr *needsBarrier=nullptr)
static void printCaptureType(OpAsmPrinter &p, Operation *op, VariableCaptureKindAttr mapCaptureType)
static LogicalResult verifyReductionModifiers(Operation *op, OperandRange reductionVars, ArrayAttr reductionMods)
Verify the per-variable modifier list has one entry per reduction variable.
static void getTaskSchedulingPointEffects(llvm::SmallVectorImpl< SideEffects::EffectInstance< MemoryEffects::Effect > > &effects)
static LogicalResult verifyNumTeamsClause(Operation *op, Value numTeamsLower, OperandRange numTeamsUpperVars)
static bool opInGlobalImplicitParallelRegion(Operation *op)
static void printTargetOpRegion(OpAsmPrinter &p, Operation *op, Region &region, ValueRange hasDeviceAddrVars, TypeRange hasDeviceAddrTypes, ValueRange hostEvalVars, TypeRange hostEvalTypes, ValueRange mapVars, TypeRange mapTypes, ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier, DenseI64ArrayAttr privateMaps)
static void printUseDeviceAddrUseDevicePtrRegion(OpAsmPrinter &p, Operation *op, Region &region, ValueRange useDeviceAddrVars, TypeRange useDeviceAddrTypes, ValueRange useDevicePtrVars, TypeRange useDevicePtrTypes)
static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars, OperandRange mapIterated)
static LogicalResult verifyPrivateVarList(OpType &op)
static ParseResult parseNumTasksClause(OpAsmParser &parser, ClauseNumTasksTypeAttr &numTasksMod, std::optional< OpAsmParser::UnresolvedOperand > &numTasks, Type &numTasksType)
LogicalResult verifyAlignment(Operation &op, std::optional< uint64_t > alignment)
Verifies align clause in allocate directive.
static ParseResult parseAlignedClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &alignedVars, SmallVectorImpl< Type > &alignedTypes, ArrayAttr &alignmentsAttr)
aligned ::= aligned ( aligned-list ) aligned-list := aligned-val | aligned-val aligned-list aligned-v...
static ParseResult parseLinearClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &linearVars, SmallVectorImpl< Type > &linearTypes, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &linearStepVars, SmallVectorImpl< Type > &linearStepTypes, ArrayAttr &linearModifiers, ArrayAttr &linearSyms)
linear ::= linear ( linear-list ) linear-list := linear-val | linear-val linear-list linear-val := ss...
static LogicalResult checkApplyeesNesting(TileOp op)
Check properties of the loop nest consisting of the transformation's applyees:
static ParseResult parseCaptureType(OpAsmParser &parser, VariableCaptureKindAttr &mapCaptureType)
static ParseResult parseTaskReductionRegion(OpAsmParser &parser, Region &region, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &taskReductionVars, SmallVectorImpl< Type > &taskReductionTypes, DenseBoolArrayAttr &taskReductionByref, ArrayAttr &taskReductionSyms)
static ParseResult parseGrainsizeClause(OpAsmParser &parser, ClauseGrainsizeTypeAttr &grainsizeMod, std::optional< OpAsmParser::UnresolvedOperand > &grainsize, Type &grainsizeType)
static ParseResult parseCopyprivate(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &copyprivateVars, SmallVectorImpl< Type > &copyprivateTypes, ArrayAttr &copyprivateSyms)
copyprivate-entry-list ::= copyprivate-entry | copyprivate-entry-list , copyprivate-entry copyprivate...
static ParseResult parseInReductionClause(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &inReductionVars, SmallVectorImpl< Type > &inReductionTypes, DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms)
Parses an in_reduction clause for an operation that does not give its list items entry block argument...
static LogicalResult verifyMapInfoDefinedArgs(Operation *op, StringRef clauseName, OperandRange vars)
static void printGrainsizeClause(OpAsmPrinter &p, Operation *op, ClauseGrainsizeTypeAttr grainsizeMod, Value grainsize, mlir::Type grainsizeType)
static ParseResult verifyScheduleModifiers(OpAsmParser &parser, SmallVectorImpl< SmallString< 12 > > &modifiers)
static Type getElementType(Type type, ArrayRef< int32_t > indices, function_ref< InFlightDiagnostic(StringRef)> emitErrorFn)
Walks the given type hierarchy with the given indices, potentially down to component granularity,...
Definition SPIRVOps.cpp:229
static bool isUnique(It begin, It end)
Definition ShardOps.cpp:159
static LogicalResult emit(SolverOp solver, const SMTEmissionOptions &options, mlir::raw_indented_ostream &stream)
Emit the SMT operations in the given 'solver' to the 'stream'.
static SmallVector< Value > getTileSizes(Location loc, x86::amx::TileType tType, RewriterBase &rewriter)
Maps the 2-dim vector shape to the two 16-bit tile sizes.
This base class exposes generic asm parser hooks, usable across the various derived parsers.
virtual ParseResult parseMinus()=0
Parse a '-' token.
@ Paren
Parens surrounding zero or more operands.
@ None
Zero or more operands with no delimiters.
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalEqual()=0
Parse a = token if present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
virtual ParseResult parseRParen()=0
Parse a ) token.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
virtual ParseResult parseOptionalColon()=0
Parse a : token if present.
virtual ParseResult parseLSquare()=0
Parse a [ token.
virtual ParseResult parseRSquare()=0
Parse a ] token.
ParseResult parseInteger(IntT &result)
Parse an integer value from the stream.
virtual ParseResult parseOptionalArrow()=0
Parse a '->' token if present.
virtual ParseResult parseLess()=0
Parse a '<' token.
virtual ParseResult parseEqual()=0
Parse a = token.
virtual ParseResult parseColonType(Type &result)=0
Parse a colon followed by a type.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual ParseResult parseOptionalComma()=0
Parse a , token if present.
virtual ParseResult parseColon()=0
Parse a : token.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseOptionalLess()=0
Parse a '<' token if present.
virtual ParseResult parseArrow()=0
Parse a '->' token.
virtual ParseResult parseLParen()=0
Parse a ( token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
virtual ParseResult parseComma()=0
Parse a , token.
virtual ParseResult parseOptionalLParen()=0
Parse a ( token if present.
ParseResult parseKeyword(StringRef keyword)
Parse a given keyword.
virtual ParseResult parseOptionalLSquare()=0
Parse a [ token if present.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
Attributes are known-constant values of operations.
Definition Attributes.h:25
This class represents an argument of a Block.
Definition Value.h:308
Block represents an ordered list of Operations.
Definition Block.h:34
MutableArrayRef< BlockArgument > BlockArgListType
Definition Block.h:110
ValueTypeRange< BlockArgListType > getArgumentTypes()
Return a range containing the types of the arguments for this block.
Definition Block.cpp:154
unsigned getNumSuccessors()
Definition Block.cpp:270
bool empty()
Definition Block.h:173
BlockArgument getArgument(unsigned i)
Definition Block.h:154
unsigned getNumArguments()
Definition Block.h:153
Operation & front()
Definition Block.h:178
SuccessorRange getSuccessors()
Definition Block.h:280
Operation & back()
Definition Block.h:177
Operation * getTerminator()
Get the terminator operation of this block.
Definition Block.cpp:249
bool mightHaveTerminator()
Return "true" if this block might have a terminator.
Definition Block.cpp:255
BlockArgListType getArguments()
Definition Block.h:112
iterator end()
Definition Block.h:169
iterator begin()
Definition Block.h:168
IntegerType getI64Type()
Definition Builders.cpp:73
IntegerAttr getI64IntegerAttr(int64_t value)
Definition Builders.cpp:120
IntegerType getIntegerType(unsigned width)
Definition Builders.cpp:75
MLIRContext * getContext() const
Definition Builders.h:56
Attr getAttr(Args &&...args)
Get or construct an instance of the attribute Attr with provided arguments.
Definition Builders.h:101
Diagnostic & append(Arg1 &&arg1, Arg2 &&arg2, Args &&...args)
Append arguments to the diagnostic.
Diagnostic & appendOp(Operation &op, const OpPrintingFlags &flags)
Append an operation with the given printing flags.
A class for computing basic dominance information.
Definition Dominance.h:143
bool dominates(Operation *a, Operation *b) const
Return true if operation A dominates operation B, i.e.
Definition Dominance.h:161
This class represents a diagnostic that is inflight and set to be reported.
Diagnostic & attachNote(std::optional< Location > noteLoc=std::nullopt)
Attaches a note to this diagnostic.
MLIRContext is the top-level object for a collection of MLIR operations.
Definition MLIRContext.h:71
NamedAttribute represents a combination of a name and an Attribute value.
Definition Attributes.h:166
StringAttr getName() const
Return the name of the attribute.
Attribute getValue() const
Return the value of the attribute.
Definition Attributes.h:181
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult parseRegion(Region &region, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region.
virtual ParseResult parseArgument(Argument &result, bool allowType=false, bool allowAttrs=false)=0
Parse a single argument with the following syntax:
virtual ParseResult parseArgumentList(SmallVectorImpl< Argument > &result, Delimiter delimiter=Delimiter::None, bool allowType=false, bool allowAttrs=false)=0
Parse zero or more arguments with a specified surrounding delimiter.
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
ParseResult resolveOperands(Operands &&operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
virtual ParseResult parseOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None, bool allowResultNumber=true, int requiredOperandCount=-1)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
virtual void printRegionArgument(BlockArgument arg, ArrayRef< NamedAttribute > argAttrs={}, bool omitType=false)=0
Print a block argument in the usual format of: ssaName : type {attr1=42} loc("here") where location p...
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
This class helps build Operations.
Definition Builders.h:210
This class represents an operand of an operation.
Definition Value.h:256
Set of flags used to control the behavior of the various IR print methods (e.g.
This class provides the API for ops that are known to be isolated from above.
This class provides the API for ops that are known to be terminators.
This class indicates that the regions associated with this op don't have terminators.
This class implements the operand iterators for the Operation class.
Definition ValueRange.h:44
type_range getType() const
Operation is the basic unit of execution within MLIR.
Definition Operation.h:87
Dialect * getDialect()
Return the dialect this operation is associated with, or nullptr if the associated dialect is not loa...
Definition Operation.h:237
Region & getRegion(unsigned index)
Returns the region held by this operation at position 'index'.
Definition Operation.h:740
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Definition Operation.h:803
Block * getBlock()
Returns the operation block that contains this operation.
Definition Operation.h:230
unsigned getNumRegions()
Returns the number of regions held by this operation.
Definition Operation.h:728
Location getLoc()
The source location the operation was defined or derived from.
Definition Operation.h:240
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition Operation.h:251
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
Definition Operation.h:255
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
Definition Operation.h:731
operand_range getOperands()
Returns an iterator on the underlying Value's.
Definition Operation.h:405
user_range getUsers()
Returns a range of all users.
Definition Operation.h:927
Region * getParentRegion()
Returns the region to which the instruction belongs.
Definition Operation.h:247
MLIRContext * getContext()
Return the context this operation is associated with.
Definition Operation.h:233
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition Region.h:26
Block & front()
Definition Region.h:65
BlockArgListType getArguments()
Definition Region.h:94
OpIterator op_begin()
Return iterators that walk the operations nested directly within this region.
Definition Region.h:178
bool isAncestor(Region *other)
Return true if this region is ancestor of the other region.
Definition Region.h:234
iterator_range< OpIterator > getOps()
Definition Region.h:180
bool empty()
Definition Region.h:60
unsigned getNumArguments()
Definition Region.h:136
Location getLoc()
Return a location for this region.
Definition Region.cpp:31
BlockArgument getArgument(unsigned i)
Definition Region.h:137
Operation * getParentOp()
Return the parent operation this region is attached to.
Definition Region.h:198
BlockListType & getBlocks()
Definition Region.h:45
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replace the results of the given (original) op with a new op that is created without verification (re...
This class represents a specific instance of an effect.
This class represents a collection of SymbolTables.
virtual Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
static Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
This class provides an abstraction over the various different ranges of value types.
Definition TypeRange.h:40
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition Types.h:74
This class provides an abstraction over the different types of ranges over Values.
Definition ValueRange.h:392
type_range getType() const
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition Value.h:96
MLIRContext * getContext() const
Utility to get the associated MLIRContext that this value is defined in.
Definition Value.h:108
Type getType() const
Return the type of this value.
Definition Value.h:105
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Definition Value.h:188
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition Value.cpp:18
A utility result that is used to signal how to proceed with an ongoing walk:
Definition WalkResult.h:29
static WalkResult advance()
Definition WalkResult.h:47
static DenseArrayAttrImpl get(MLIRContext *context, ArrayRef< bool > content)
bool isReachableFromEntry(Block *a) const
Return true if the specified block is reachable from the entry block of its region.
Operation * getOwner() const
Return the owner of this operand.
Definition UseDefLists.h:38
TargetEnterDataOperands TargetEnterExitUpdateDataOperands
omp.target_enter_data, omp.target_exit_data and omp.target_update take the same clauses,...
std::tuple< NewCliOp, OpOperand *, OpOperand * > decodeCli(mlir::Value cli)
Find the omp.new_cli, generator, and consumer of a canonical loop info.
ClauseProcBindKind convertProcBindKind(llvm::omp::ProcBindKind kind)
Convert a proc_bind kind from the LLVM frontend enum to the corresponding OpenMP dialect enum.
detail::InFlightRemark failed(Location loc, RemarkOpts opts)
Report an optimization remark that failed.
Definition Remarks.h:853
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition Matchers.h:490
detail::DenseArrayAttrImpl< int64_t > DenseI64ArrayAttr
function_ref< void(Value, StringRef)> OpAsmSetValueNameFn
A functor used to set the name of the start of a result group of an operation.
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
Definition Utils.cpp:311
llvm::DenseSet< ValueT, ValueInfoT > DenseSet
Definition LLVM.h:125
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
bool isPure(Operation *op)
Returns true if the given operation is pure, i.e., is speculatable that does not touch memory.
detail::constant_int_predicate_matcher m_Zero()
Matches a constant scalar / vector splat / tensor splat integer zero.
Definition Matchers.h:442
std::conditional_t< std::is_same_v< Ty, mlir::Type >, mlir::Value, detail::TypedValue< Ty > > TypedValue
If Ty is mlir::Type this will select Value instead of having a wrapper around it.
Definition Value.h:496
llvm::TypeSwitch< T, ResultT > TypeSwitch
Definition LLVM.h:147
SmallVector< Loops, 8 > tile(ArrayRef< scf::ForOp > forOps, ArrayRef< Value > sizes, ArrayRef< scf::ForOp > targets)
Performs tiling fo imperfectly nested loops (with interchange) by strip-mining the forOps by sizes an...
Definition Utils.cpp:1389
detail::DenseArrayAttrImpl< bool > DenseBoolArrayAttr
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Definition Matchers.h:369
function_ref< void(Block *, StringRef)> OpAsmSetBlockNameFn
A functor used to set the name of blocks in regions directly nested under an operation.
This is the representation of an operand reference.
This class provides APIs and verifiers for ops with regions having a single block.
This represents an operation in an abstracted form, suitable for use with the builder APIs.
T & getOrAddProperties()
Get (or create) the properties of the provided type to be set on the operation on creation.
void addOperands(ValueRange newOperands)
void addAttributes(ArrayRef< NamedAttribute > newAttributes)
Add an array of named attributes.
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
void addTypes(ArrayRef< Type > newTypes)
Region * addRegion()
Create a region that should be attached to the operation.
Extended TargetOperands with kernel_type attribute.
TargetExecModeAttr kernelType
Kernel execution mode for the target region.