OpenMPDialect.cpp

Go to the documentation of this file.

//===- OpenMPDialect.cpp - MLIR Dialect for OpenMP implementation ---------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements the OpenMP dialect and its operations.
//
//===----------------------------------------------------------------------===//
 
#include "mlir/Dialect/OpenMP/OpenMPDialect.h"
#include "mlir/Conversion/ConvertToLLVM/ToLLVMInterface.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
#include "mlir/Dialect/OpenMP/OpenMPClauseOperands.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/IR/OperationSupport.h"
#include "mlir/IR/SymbolTable.h"
#include "mlir/Interfaces/FoldInterfaces.h"
#include "mlir/Interfaces/SideEffectInterfaces.h"
 
#include "llvm/ADT/ArrayRef.h"
#include "llvm/ADT/PostOrderIterator.h"
#include "llvm/ADT/STLExtras.h"
#include "llvm/ADT/STLForwardCompat.h"
#include "llvm/ADT/SmallString.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/ADT/StringRef.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/ADT/bit.h"
#include "llvm/Support/InterleavedRange.h"
#include <cstddef>
#include <iterator>
#include <optional>
#include <variant>
 
#include "mlir/Dialect/OpenMP/OpenMPOpsDialect.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPOpsEnums.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPOpsInterfaces.cpp.inc"
#include "mlir/Dialect/OpenMP/OpenMPTypeInterfaces.cpp.inc"
 
using namespace mlir;
using namespace mlir::omp;
 
static ArrayAttr makeArrayAttr(MLIRContext *context,
                               llvm::ArrayRef<Attribute> attrs) {
  return attrs.empty() ? nullptr : ArrayAttr::get(context, attrs);
}
 
static DenseBoolArrayAttr
makeDenseBoolArrayAttr(MLIRContext *ctx, const ArrayRef<bool> boolArray) {
  return boolArray.empty() ? nullptr : DenseBoolArrayAttr::get(ctx, boolArray);
}
 
static DenseI64ArrayAttr
makeDenseI64ArrayAttr(MLIRContext *ctx, const ArrayRef<int64_t> intArray) {
  return intArray.empty() ? nullptr : DenseI64ArrayAttr::get(ctx, intArray);
}
 
namespace {
struct MemRefPointerLikeModel
    : public PointerLikeType::ExternalModel<MemRefPointerLikeModel,
                                            MemRefType> {
  Type getElementType(Type pointer) const {
    return llvm::cast<MemRefType>(pointer).getElementType();
  }
};
 
struct LLVMPointerPointerLikeModel
    : public PointerLikeType::ExternalModel<LLVMPointerPointerLikeModel,
                                            LLVM::LLVMPointerType> {
  Type getElementType(Type pointer) const { return Type(); }
};
} // namespace
 
/// Generate a name of a canonical loop nest of the format
/// `<prefix>(_r<idx>_s<idx>)*`. Hereby, `_r<idx>` identifies the region
/// argument index of an operation that has multiple regions, if the operation
/// has multiple regions.
/// `_s<idx>` identifies the position of an operation within a region, where
/// only operations that may potentially contain loops ("container operations"
/// i.e. have region arguments) are counted. Again, it is omitted if there is
/// only one such operation in a region. If there are canonical loops nested
/// inside each other, also may also use the format `_d<num>` where <num> is the
/// nesting depth of the loop.
///
/// The generated name is a best-effort to make canonical loop unique within an
/// SSA namespace. This also means that regions with IsolatedFromAbove property
/// do not consider any parents or siblings.
static std::string generateLoopNestingName(StringRef prefix,
                                           CanonicalLoopOp op) {
  struct Component {
    /// If true, this component describes a region operand of an operation (the
    /// operand's owner) If false, this component describes an operation located
    /// in a parent region
    bool isRegionArgOfOp;
    bool skip = false;
    bool isUnique = false;
 
    size_t idx;
    Operation *op;
    Region *parentRegion;
    size_t loopDepth;
 
    Operation *&getOwnerOp() {
      assert(isRegionArgOfOp && "Must describe a region operand");
      return op;
    }
    size_t &getArgIdx() {
      assert(isRegionArgOfOp && "Must describe a region operand");
      return idx;
    }
 
    Operation *&getContainerOp() {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return op;
    }
    size_t &getOpPos() {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return idx;
    }
    bool isLoopOp() const {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return isa<CanonicalLoopOp>(op);
    }
    Region *&getParentRegion() {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return parentRegion;
    }
    size_t &getLoopDepth() {
      assert(!isRegionArgOfOp && "Must describe a operation of a region");
      return loopDepth;
    }
 
    void skipIf(bool v = true) { skip = skip || v; }
  };
 
  // List of ancestors, from inner to outer.
  // Alternates between
  //  * region argument of an operation
  //  * operation within a region
  SmallVector<Component> components;
 
  // Gather a list of parent regions and operations, and the position within
  // their parent
  Operation *o = op.getOperation();
  while (o) {
    // Operation within a region
    Region *r = o->getParentRegion();
    if (!r)
      break;
 
    llvm::ReversePostOrderTraversal<Block *> traversal(&r->getBlocks().front());
    size_t idx = 0;
    bool found = false;
    size_t sequentialIdx = -1;
    bool isOnlyContainerOp = true;
    for (Block *b : traversal) {
      for (Operation &op : *b) {
        if (&op == o && !found) {
          sequentialIdx = idx;
          found = true;
        }
        if (op.getNumRegions()) {
          idx += 1;
          if (idx > 1)
            isOnlyContainerOp = false;
        }
        if (found && !isOnlyContainerOp)
          break;
      }
    }
 
    Component &containerOpInRegion = components.emplace_back();
    containerOpInRegion.isRegionArgOfOp = false;
    containerOpInRegion.isUnique = isOnlyContainerOp;
    containerOpInRegion.getContainerOp() = o;
    containerOpInRegion.getOpPos() = sequentialIdx;
    containerOpInRegion.getParentRegion() = r;
 
    Operation *parent = r->getParentOp();
 
    // Region argument of an operation
    Component &regionArgOfOperation = components.emplace_back();
    regionArgOfOperation.isRegionArgOfOp = true;
    regionArgOfOperation.isUnique = true;
    regionArgOfOperation.getArgIdx() = 0;
    regionArgOfOperation.getOwnerOp() = parent;
 
    // The IsolatedFromAbove trait of the parent operation implies that each
    // individual region argument has its own separate namespace, so no
    // ambiguity.
    if (!parent || parent->hasTrait<mlir::OpTrait::IsIsolatedFromAbove>())
      break;
 
    // Component only needed if operation has multiple region operands. Region
    // arguments may be optional, but we currently do not consider this.
    if (parent->getRegions().size() > 1) {
      auto getRegionIndex = [](Operation *o, Region *r) {
        for (auto [idx, region] : llvm::enumerate(o->getRegions())) {
          if (&region == r)
            return idx;
        }
        llvm_unreachable("Region not child of its parent operation");
      };
      regionArgOfOperation.isUnique = false;
      regionArgOfOperation.getArgIdx() = getRegionIndex(parent, r);
    }
 
    // next parent
    o = parent;
  }
 
  // Determine whether a region-argument component is not needed
  for (Component &c : components)
    c.skipIf(c.isRegionArgOfOp && c.isUnique);
 
  // Find runs of nested loops and determine each loop's depth in the loop nest
  size_t numSurroundingLoops = 0;
  for (Component &c : llvm::reverse(components)) {
    if (c.skip)
      continue;
 
    // non-skipped multi-argument operands interrupt the loop nest
    if (c.isRegionArgOfOp) {
      numSurroundingLoops = 0;
      continue;
    }
 
    // Multiple loops in a region means each of them is the outermost loop of a
    // new loop nest
    if (!c.isUnique)
      numSurroundingLoops = 0;
 
    c.getLoopDepth() = numSurroundingLoops;
 
    // Next loop is surrounded by one more loop
    if (isa<CanonicalLoopOp>(c.getContainerOp()))
      numSurroundingLoops += 1;
  }
 
  // In loop nests, skip all but the innermost loop that contains the depth
  // number
  bool isLoopNest = false;
  for (Component &c : components) {
    if (c.skip || c.isRegionArgOfOp)
      continue;
 
    if (!isLoopNest && c.getLoopDepth() >= 1) {
      // Innermost loop of a loop nest of at least two loops
      isLoopNest = true;
    } else if (isLoopNest) {
      // Non-innermost loop of a loop nest
      c.skipIf(c.isUnique);
 
      // If there is no surrounding loop left, this must have been the outermost
      // loop; leave loop-nest mode for the next iteration
      if (c.getLoopDepth() == 0)
        isLoopNest = false;
    }
  }
 
  // Skip non-loop unambiguous regions (but they should interrupt loop nests, so
  // we mark them as skipped only after computing loop nests)
  for (Component &c : components)
    c.skipIf(!c.isRegionArgOfOp && c.isUnique &&
             !isa<CanonicalLoopOp>(c.getContainerOp()));
 
  // Components can be skipped if they are already disambiguated by their parent
  // (or does not have a parent)
  bool newRegion = true;
  for (Component &c : llvm::reverse(components)) {
    c.skipIf(newRegion && c.isUnique);
 
    // non-skipped components disambiguate unique children
    if (!c.skip)
      newRegion = true;
 
    // ...except canonical loops that need a suffix for each nest
    if (!c.isRegionArgOfOp && c.getContainerOp())
      newRegion = false;
  }
 
  // Compile the nesting name string
  SmallString<64> Name{prefix};
  llvm::raw_svector_ostream NameOS(Name);
  for (auto &c : llvm::reverse(components)) {
    if (c.skip)
      continue;
 
    if (c.isRegionArgOfOp)
      NameOS << "_r" << c.getArgIdx();
    else if (c.getLoopDepth() >= 1)
      NameOS << "_d" << c.getLoopDepth();
    else
      NameOS << "_s" << c.getOpPos();
  }
 
  return NameOS.str().str();
}
 
void OpenMPDialect::initialize() {
  addOperations<
#define GET_OP_LIST
#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
      >();
  addAttributes<
#define GET_ATTRDEF_LIST
#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
      >();
  addTypes<
#define GET_TYPEDEF_LIST
#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"
      >();
 
  declarePromisedInterface<ConvertToLLVMPatternInterface, OpenMPDialect>();
 
  MemRefType::attachInterface<MemRefPointerLikeModel>(*getContext());
  LLVM::LLVMPointerType::attachInterface<LLVMPointerPointerLikeModel>(
      *getContext());
 
  // Attach default offload module interface to module op to access
  // offload functionality through
  mlir::ModuleOp::attachInterface<mlir::omp::OffloadModuleDefaultModel>(
      *getContext());
 
  // Attach default declare target interfaces to operations which can be marked
  // as declare target (Global Operations and Functions/Subroutines in dialects
  // that Fortran (or other languages that lower to MLIR) translates too
  mlir::LLVM::GlobalOp::attachInterface<
      mlir::omp::DeclareTargetDefaultModel<mlir::LLVM::GlobalOp>>(
      *getContext());
  mlir::LLVM::LLVMFuncOp::attachInterface<
      mlir::omp::DeclareTargetDefaultModel<mlir::LLVM::LLVMFuncOp>>(
      *getContext());
  mlir::func::FuncOp::attachInterface<
      mlir::omp::DeclareTargetDefaultModel<mlir::func::FuncOp>>(*getContext());
}
 
//===----------------------------------------------------------------------===//
// Dialect operation attribute verification
//===----------------------------------------------------------------------===//
 
static LogicalResult verifyDeclareTargetAttr(Operation *op, Attribute attr) {
  if (!isa<DeclareTargetInterface>(op))
    return op->emitError() << "omp.declare_target can only be applied to "
                              "DeclareTargetInterface ops";
 
  auto declareTargetAttr = dyn_cast<DeclareTargetAttr>(attr);
  if (!declareTargetAttr)
    return op->emitError()
           << "omp.declare_target must be an #omp.declaretarget attribute";
 
  if (isa<mlir::FunctionOpInterface>(op)) {
    if (declareTargetAttr.getAutomap())
      return op->emitOpError()
             << "omp.declare_target 'automap' is not valid on functions";
 
    // TODO: Disallow the `local` clause (OpenMP 6.0).
    if (declareTargetAttr.getCaptureClause() ==
        mlir::omp::DeclareTargetCaptureClause::link)
      return op->emitOpError()
             << "omp.declare_target 'link' is not valid on functions";
  } else {
    // TODO: Disallow the `indirect` clause (OpenMP 5.1).
    if (declareTargetAttr.getImplicit())
      return op->emitOpError()
             << "omp.declare_target 'implicit' is only valid on functions";
  }
  return success();
}
 
LogicalResult
OpenMPDialect::verifyOperationAttribute(Operation *op,
                                        NamedAttribute attribute) {
  if (attribute.getName() == "omp.declare_target")
    return verifyDeclareTargetAttr(op, attribute.getValue());
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for Allocate Clause
//===----------------------------------------------------------------------===//
 
/// Parse an allocate clause with allocators and a list of operands with types.
///
/// allocate-operand-list :: = allocate-operand |
///                            allocator-operand `,` allocate-operand-list
/// allocate-operand :: = ssa-id-and-type -> ssa-id-and-type
/// ssa-id-and-type ::= ssa-id `:` type
static ParseResult parseAllocateAndAllocator(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocateVars,
    SmallVectorImpl<Type> &allocateTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &allocatorVars,
    SmallVectorImpl<Type> &allocatorTypes) {
 
  return parser.parseCommaSeparatedList([&]() {
    OpAsmParser::UnresolvedOperand operand;
    Type type;
    if (parser.parseOperand(operand) || parser.parseColonType(type))
      return failure();
    allocatorVars.push_back(operand);
    allocatorTypes.push_back(type);
    if (parser.parseArrow())
      return failure();
    if (parser.parseOperand(operand) || parser.parseColonType(type))
      return failure();
 
    allocateVars.push_back(operand);
    allocateTypes.push_back(type);
    return success();
  });
}
 
/// Print allocate clause
static void printAllocateAndAllocator(OpAsmPrinter &p, Operation *op,
                                      OperandRange allocateVars,
                                      TypeRange allocateTypes,
                                      OperandRange allocatorVars,
                                      TypeRange allocatorTypes) {
  for (unsigned i = 0; i < allocateVars.size(); ++i) {
    std::string separator = i == allocateVars.size() - 1 ? "" : ", ";
    p << allocatorVars[i] << " : " << allocatorTypes[i] << " -> ";
    p << allocateVars[i] << " : " << allocateTypes[i] << separator;
  }
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for a clause attribute (StringEnumAttr)
//===----------------------------------------------------------------------===//
 
template <typename ClauseAttr>
static ParseResult parseClauseAttr(AsmParser &parser, ClauseAttr &attr) {
  using ClauseT = decltype(std::declval<ClauseAttr>().getValue());
  StringRef enumStr;
  SMLoc loc = parser.getCurrentLocation();
  if (parser.parseKeyword(&enumStr))
    return failure();
  if (std::optional<ClauseT> enumValue = symbolizeEnum<ClauseT>(enumStr)) {
    attr = ClauseAttr::get(parser.getContext(), *enumValue);
    return success();
  }
  return parser.emitError(loc, "invalid clause value: '") << enumStr << "'";
}
 
template <typename ClauseAttr>
static void printClauseAttr(OpAsmPrinter &p, Operation *op, ClauseAttr attr) {
  p << stringifyEnum(attr.getValue());
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for Linear Clause
//===----------------------------------------------------------------------===//
 
/// linear ::= `linear` `(` linear-list `)`
/// linear-list := linear-val | linear-val linear-list
/// linear-val := ssa-id-and-type `=` ssa-id-and-type
///           | `val` `(` ssa-id-and-type `=` ssa-id-and-type `)`
///           | `ref` `(` ssa-id-and-type `=` ssa-id-and-type `)`
///           | `uval` `(` ssa-id-and-type `=` ssa-id-and-type `)`
static ParseResult parseLinearClause(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
    SmallVectorImpl<Type> &linearTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
    SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearModifiers,
    ArrayAttr &linearSyms) {
  SmallVector<Attribute> modifiers;
  auto result = parser.parseCommaSeparatedList([&]() {
    OpAsmParser::UnresolvedOperand var;
    Type type, stepType;
    OpAsmParser::UnresolvedOperand stepVar;
 
    std::optional<omp::LinearModifier> linearModifier;
    if (succeeded(parser.parseOptionalKeyword("val"))) {
      linearModifier = omp::LinearModifier::val;
    } else if (succeeded(parser.parseOptionalKeyword("ref"))) {
      linearModifier = omp::LinearModifier::ref;
    } else if (succeeded(parser.parseOptionalKeyword("uval"))) {
      linearModifier = omp::LinearModifier::uval;
    }
 
    bool hasLinearModifierParens = linearModifier.has_value();
    if (hasLinearModifierParens && parser.parseLParen())
      return failure();
 
    if (parser.parseOperand(var) || parser.parseColonType(type) ||
        parser.parseEqual() || parser.parseOperand(stepVar) ||
        parser.parseColonType(stepType))
      return failure();
 
    if (hasLinearModifierParens && parser.parseRParen())
      return failure();
 
    linearVars.push_back(var);
    linearTypes.push_back(type);
    linearStepVars.push_back(stepVar);
    linearStepTypes.push_back(stepType);
    if (linearModifier) {
      modifiers.push_back(
          omp::LinearModifierAttr::get(parser.getContext(), *linearModifier));
    } else {
      modifiers.push_back(UnitAttr::get(parser.getContext()));
    }
    return success();
  });
  if (failed(result))
    return failure();
  linearModifiers = ArrayAttr::get(parser.getContext(), modifiers);
  return success();
}
 
/// Print Linear Clause. `linearSyms` is only used by ops with a region, which
/// print the linear clause with their entry block arguments.
static void printLinearClause(OpAsmPrinter &p, Operation *op,
                              ValueRange linearVars, TypeRange linearTypes,
                              ValueRange linearStepVars, TypeRange stepVarTypes,
                              ArrayAttr linearModifiers, ArrayAttr linearSyms) {
  size_t linearVarsSize = linearVars.size();
  for (unsigned i = 0; i < linearVarsSize; ++i) {
    if (i != 0)
      p << ", ";
    // Print modifier keyword wrapper if present.
    Attribute modAttr = linearModifiers ? linearModifiers[i] : nullptr;
    auto mod = modAttr ? dyn_cast<omp::LinearModifierAttr>(modAttr) : nullptr;
    if (mod) {
      p << omp::stringifyLinearModifier(mod.getValue()) << "(";
    }
    p << linearVars[i] << " : " << linearTypes[i];
    p << " = " << linearStepVars[i] << " : " << stepVarTypes[i];
    if (mod)
      p << ")";
  }
}
 
//===----------------------------------------------------------------------===//
// Verifier for Linear modifier
//===----------------------------------------------------------------------===//
 
/// OpenMP 5.2, Section 5.4.6: "A linear-modifier may be specified as ref or
/// uval only on a declare simd directive."
/// Also verifies that modifier count matches variable count.
static LogicalResult
verifyLinearModifiers(Operation *op, std::optional<ArrayAttr> linearModifiers,
                      OperandRange linearVars, bool isDeclareSimd = false) {
  if (!linearModifiers)
    return success();
  if (linearModifiers->size() != linearVars.size())
    return op->emitOpError()
           << "expected as many linear modifiers as linear variables";
  if (!isDeclareSimd) {
    for (Attribute attr : *linearModifiers) {
      if (!attr)
        continue;
      auto modAttr = dyn_cast<omp::LinearModifierAttr>(attr);
      if (!modAttr)
        continue;
      omp::LinearModifier mod = modAttr.getValue();
      if (mod == omp::LinearModifier::ref || mod == omp::LinearModifier::uval)
        return op->emitOpError()
               << "linear modifier '" << omp::stringifyLinearModifier(mod)
               << "' may only be specified on a declare simd directive";
    }
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// Verifier for Nontemporal Clause
//===----------------------------------------------------------------------===//
 
static LogicalResult verifyNontemporalClause(Operation *op,
                                             OperandRange nontemporalVars) {
 
  // Check if each var is unique - OpenMP 5.0 -> 2.9.3.1 section
  DenseSet<Value> nontemporalItems;
  for (const auto &it : nontemporalVars)
    if (!nontemporalItems.insert(it).second)
      return op->emitOpError() << "nontemporal variable used more than once";
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Parser, verifier and printer for Aligned Clause
//===----------------------------------------------------------------------===//
static LogicalResult verifyAlignedClause(Operation *op,
                                         std::optional<ArrayAttr> alignments,
                                         OperandRange alignedVars) {
  // Check if number of alignment values equals to number of aligned variables
  if (!alignedVars.empty()) {
    if (!alignments || alignments->size() != alignedVars.size())
      return op->emitOpError()
             << "expected as many alignment values as aligned variables";
  } else {
    if (alignments)
      return op->emitOpError() << "unexpected alignment values attribute";
    return success();
  }
 
  // Check if each var is aligned only once - OpenMP 4.5 -> 2.8.1 section
  DenseSet<Value> alignedItems;
  for (auto it : alignedVars)
    if (!alignedItems.insert(it).second)
      return op->emitOpError() << "aligned variable used more than once";
 
  if (!alignments)
    return success();
 
  // Check if all alignment values are positive - OpenMP 4.5 -> 2.8.1 section
  for (unsigned i = 0; i < (*alignments).size(); ++i) {
    if (auto intAttr = llvm::dyn_cast<IntegerAttr>((*alignments)[i])) {
      if (intAttr.getValue().sle(0))
        return op->emitOpError() << "alignment should be greater than 0";
    } else {
      return op->emitOpError() << "expected integer alignment";
    }
  }
 
  return success();
}
 
/// aligned ::= `aligned` `(` aligned-list `)`
/// aligned-list := aligned-val | aligned-val aligned-list
/// aligned-val := ssa-id-and-type `->` alignment
static ParseResult
parseAlignedClause(OpAsmParser &parser,
                   SmallVectorImpl<OpAsmParser::UnresolvedOperand> &alignedVars,
                   SmallVectorImpl<Type> &alignedTypes,
                   ArrayAttr &alignmentsAttr) {
  SmallVector<Attribute> alignmentVec;
  if (failed(parser.parseCommaSeparatedList([&]() {
        if (parser.parseOperand(alignedVars.emplace_back()) ||
            parser.parseColonType(alignedTypes.emplace_back()) ||
            parser.parseArrow() ||
            parser.parseAttribute(alignmentVec.emplace_back())) {
          return failure();
        }
        return success();
      })))
    return failure();
  SmallVector<Attribute> alignments(alignmentVec.begin(), alignmentVec.end());
  alignmentsAttr = ArrayAttr::get(parser.getContext(), alignments);
  return success();
}
 
/// Print Aligned Clause
static void printAlignedClause(OpAsmPrinter &p, Operation *op,
                               ValueRange alignedVars, TypeRange alignedTypes,
                               std::optional<ArrayAttr> alignments) {
  for (unsigned i = 0; i < alignedVars.size(); ++i) {
    if (i != 0)
      p << ", ";
    p << alignedVars[i] << " : " << alignedVars[i].getType();
    p << " -> " << (*alignments)[i];
  }
}
 
static LogicalResult verifyAllocateClause(
    Operation *op, ValueRange allocateVars, ValueRange allocatorVars,
    DenseI64ArrayAttr allocateAlignments,
    DenseI64ArrayAttr allocatePrivateIndices, ValueRange privateVars = {},
    ArrayAttr privateSyms = nullptr, bool requirePrivateIndices = false) {
  if (allocateVars.size() != allocatorVars.size())
    return op->emitError(
        "expected equal sizes for allocate and allocator variables");
 
  if (allocateVars.empty()) {
    if (allocateAlignments)
      return op->emitError(
          "unexpected allocate alignments without allocate variables");
    if (allocatePrivateIndices)
      return op->emitError(
          "unexpected allocate private indices without allocate variables");
    return success();
  }
 
  if (allocateAlignments) {
    ArrayRef<int64_t> alignments = allocateAlignments.asArrayRef();
    if (alignments.size() != allocateVars.size())
      return op->emitError(
          "expected as many allocate alignments as allocate variables");
    for (int64_t alignment : alignments) {
      if (alignment < 0)
        return op->emitError("expected non-negative allocate alignments");
      if (alignment != 0 && (alignment & (alignment - 1)) != 0)
        return op->emitError(
            "expected positive allocate alignments to be powers of two");
    }
  }
 
  if (!allocatePrivateIndices) {
    if (requirePrivateIndices)
      return op->emitError(
          "expected an allocate private index for each allocate variable");
    return success();
  }
 
  ArrayRef<int64_t> indices = allocatePrivateIndices.asArrayRef();
  if (indices.size() != allocateVars.size())
    return op->emitError(
        "expected as many allocate private indices as allocate variables");
 
  DenseSet<int64_t> usedPrivateSlots;
  for (auto [allocateVar, privateIndex] :
       llvm::zip_equal(allocateVars, indices)) {
    if (privateIndex < 0 ||
        static_cast<uint64_t>(privateIndex) >= privateVars.size())
      return op->emitError("allocate private index is out of range");
    if (!usedPrivateSlots.insert(privateIndex).second)
      return op->emitError(
          "allocate private index refers to a private variable more than once");
 
    Value privateVar = privateVars[privateIndex];
    if (allocateVar.getType() != privateVar.getType())
      return op->emitError()
             << "type mismatch between allocate variable and private variable "
                "at index "
             << privateIndex;
    if (allocateVar != privateVar)
      return op->emitError()
             << "allocate variable does not match private variable at index "
             << privateIndex;
 
    if (!privateSyms ||
        static_cast<uint64_t>(privateIndex) >= privateSyms.size())
      return op->emitError(
          "allocate private index does not have a privatizer symbol");
 
    auto privateSym = dyn_cast<SymbolRefAttr>(privateSyms[privateIndex]);
    if (!privateSym)
      return op->emitError(
          "allocate private index does not reference a privatizer symbol");
    PrivateClauseOp privatizer =
        SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, privateSym);
    if (!privatizer)
      return op->emitError() << "failed to lookup privatizer op with symbol: '"
                             << privateSym << "'";
    if (privatizer.getDataSharingType() != DataSharingClauseType::Private &&
        privatizer.getDataSharingType() != DataSharingClauseType::FirstPrivate)
      return op->emitError(
          "allocate private index must refer to private or firstprivate "
          "storage");
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Parser, printer and verifier for Schedule Clause
//===----------------------------------------------------------------------===//
 
static ParseResult
verifyScheduleModifiers(OpAsmParser &parser,
                        SmallVectorImpl<SmallString<12>> &modifiers) {
  if (modifiers.size() > 2)
    return parser.emitError(parser.getNameLoc()) << " unexpected modifier(s)";
  for (const auto &mod : modifiers) {
    // Translate the string. If it has no value, then it was not a valid
    // modifier!
    auto symbol = symbolizeScheduleModifier(mod);
    if (!symbol)
      return parser.emitError(parser.getNameLoc())
             << " unknown modifier type: " << mod;
  }
 
  // If we have one modifier that is "simd", then stick a "none" modiifer in
  // index 0.
  if (modifiers.size() == 1) {
    if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd) {
      modifiers.push_back(modifiers[0]);
      modifiers[0] = stringifyScheduleModifier(ScheduleModifier::none);
    }
  } else if (modifiers.size() == 2) {
    // If there are two modifier:
    // First modifier should not be simd, second one should be simd
    if (symbolizeScheduleModifier(modifiers[0]) == ScheduleModifier::simd ||
        symbolizeScheduleModifier(modifiers[1]) != ScheduleModifier::simd)
      return parser.emitError(parser.getNameLoc())
             << " incorrect modifier order";
  }
  return success();
}
 
/// schedule ::= `schedule` `(` sched-list `)`
/// sched-list ::= sched-val | sched-val sched-list |
///                sched-val `,` sched-modifier
/// sched-val ::= sched-with-chunk | sched-wo-chunk
/// sched-with-chunk ::= sched-with-chunk-types (`=` ssa-id-and-type)?
/// sched-with-chunk-types ::= `static` | `dynamic` | `guided`
/// sched-wo-chunk ::=  `auto` | `runtime`
/// sched-modifier ::=  sched-mod-val | sched-mod-val `,` sched-mod-val
/// sched-mod-val ::=  `monotonic` | `nonmonotonic` | `simd` | `none`
static ParseResult
parseScheduleClause(OpAsmParser &parser, ClauseScheduleKindAttr &scheduleAttr,
                    ScheduleModifierAttr &scheduleMod, UnitAttr &scheduleSimd,
                    std::optional<OpAsmParser::UnresolvedOperand> &chunkSize,
                    Type &chunkType) {
  StringRef keyword;
  if (parser.parseKeyword(&keyword))
    return failure();
  std::optional<mlir::omp::ClauseScheduleKind> schedule =
      symbolizeClauseScheduleKind(keyword);
  if (!schedule)
    return parser.emitError(parser.getNameLoc()) << " expected schedule kind";
 
  scheduleAttr = ClauseScheduleKindAttr::get(parser.getContext(), *schedule);
  switch (*schedule) {
  case ClauseScheduleKind::Static:
  case ClauseScheduleKind::Dynamic:
  case ClauseScheduleKind::Guided:
    if (succeeded(parser.parseOptionalEqual())) {
      chunkSize = OpAsmParser::UnresolvedOperand{};
      if (parser.parseOperand(*chunkSize) || parser.parseColonType(chunkType))
        return failure();
    } else {
      chunkSize = std::nullopt;
    }
    break;
  case ClauseScheduleKind::Auto:
  case ClauseScheduleKind::Runtime:
  case ClauseScheduleKind::Distribute:
    chunkSize = std::nullopt;
  }
 
  // If there is a comma, we have one or more modifiers..
  SmallVector<SmallString<12>> modifiers;
  while (succeeded(parser.parseOptionalComma())) {
    StringRef mod;
    if (parser.parseKeyword(&mod))
      return failure();
    modifiers.push_back(mod);
  }
 
  if (verifyScheduleModifiers(parser, modifiers))
    return failure();
 
  if (!modifiers.empty()) {
    SMLoc loc = parser.getCurrentLocation();
    if (std::optional<ScheduleModifier> mod =
            symbolizeScheduleModifier(modifiers[0])) {
      scheduleMod = ScheduleModifierAttr::get(parser.getContext(), *mod);
    } else {
      return parser.emitError(loc, "invalid schedule modifier");
    }
    // Only SIMD attribute is allowed here!
    if (modifiers.size() > 1) {
      assert(symbolizeScheduleModifier(modifiers[1]) == ScheduleModifier::simd);
      scheduleSimd = UnitAttr::get(parser.getBuilder().getContext());
    }
  }
 
  return success();
}
 
/// Print schedule clause
static void printScheduleClause(OpAsmPrinter &p, Operation *op,
                                ClauseScheduleKindAttr scheduleKind,
                                ScheduleModifierAttr scheduleMod,
                                UnitAttr scheduleSimd, Value scheduleChunk,
                                Type scheduleChunkType) {
  p << stringifyClauseScheduleKind(scheduleKind.getValue());
  if (scheduleChunk)
    p << " = " << scheduleChunk << " : " << scheduleChunk.getType();
  if (scheduleMod)
    p << ", " << stringifyScheduleModifier(scheduleMod.getValue());
  if (scheduleSimd)
    p << ", simd";
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for Order Clause
//===----------------------------------------------------------------------===//
 
// order ::= `order` `(` [order-modifier ':'] concurrent `)`
// order-modifier ::= reproducible | unconstrained
static ParseResult parseOrderClause(OpAsmParser &parser,
                                    ClauseOrderKindAttr &order,
                                    OrderModifierAttr &orderMod) {
  StringRef enumStr;
  SMLoc loc = parser.getCurrentLocation();
  if (parser.parseKeyword(&enumStr))
    return failure();
  if (std::optional<OrderModifier> enumValue =
          symbolizeOrderModifier(enumStr)) {
    orderMod = OrderModifierAttr::get(parser.getContext(), *enumValue);
    if (parser.parseOptionalColon())
      return failure();
    loc = parser.getCurrentLocation();
    if (parser.parseKeyword(&enumStr))
      return failure();
  }
  if (std::optional<ClauseOrderKind> enumValue =
          symbolizeClauseOrderKind(enumStr)) {
    order = ClauseOrderKindAttr::get(parser.getContext(), *enumValue);
    return success();
  }
  return parser.emitError(loc, "invalid clause value: '") << enumStr << "'";
}
 
static void printOrderClause(OpAsmPrinter &p, Operation *op,
                             ClauseOrderKindAttr order,
                             OrderModifierAttr orderMod) {
  if (orderMod)
    p << stringifyOrderModifier(orderMod.getValue()) << ":";
  if (order)
    p << stringifyClauseOrderKind(order.getValue());
}
 
template <typename ClauseTypeAttr, typename ClauseType>
static ParseResult
parseGranularityClause(OpAsmParser &parser, ClauseTypeAttr &prescriptiveness,
                       std::optional<OpAsmParser::UnresolvedOperand> &operand,
                       Type &operandType,
                       std::optional<ClauseType> (*symbolizeClause)(StringRef),
                       StringRef clauseName) {
  StringRef enumStr;
  if (succeeded(parser.parseOptionalKeyword(&enumStr))) {
    if (std::optional<ClauseType> enumValue = symbolizeClause(enumStr)) {
      prescriptiveness = ClauseTypeAttr::get(parser.getContext(), *enumValue);
      if (parser.parseComma())
        return failure();
    } else {
      return parser.emitError(parser.getCurrentLocation())
             << "invalid " << clauseName << " modifier : '" << enumStr << "'";
      ;
    }
  }
 
  OpAsmParser::UnresolvedOperand var;
  if (succeeded(parser.parseOperand(var))) {
    operand = var;
  } else {
    return parser.emitError(parser.getCurrentLocation())
           << "expected " << clauseName << " operand";
  }
 
  if (operand.has_value()) {
    if (parser.parseColonType(operandType))
      return failure();
  }
 
  return success();
}
 
template <typename ClauseTypeAttr, typename ClauseType>
static void
printGranularityClause(OpAsmPrinter &p, Operation *op,
                       ClauseTypeAttr prescriptiveness, Value operand,
                       mlir::Type operandType,
                       StringRef (*stringifyClauseType)(ClauseType)) {
 
  if (prescriptiveness)
    p << stringifyClauseType(prescriptiveness.getValue()) << ", ";
 
  if (operand)
    p << operand << ": " << operandType;
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for grainsize Clause
//===----------------------------------------------------------------------===//
 
// grainsize ::= `grainsize` `(` [strict ':'] grain-size `)`
static ParseResult
parseGrainsizeClause(OpAsmParser &parser, ClauseGrainsizeTypeAttr &grainsizeMod,
                     std::optional<OpAsmParser::UnresolvedOperand> &grainsize,
                     Type &grainsizeType) {
  return parseGranularityClause<ClauseGrainsizeTypeAttr, ClauseGrainsizeType>(
      parser, grainsizeMod, grainsize, grainsizeType,
      &symbolizeClauseGrainsizeType, "grainsize");
}
 
static void printGrainsizeClause(OpAsmPrinter &p, Operation *op,
                                 ClauseGrainsizeTypeAttr grainsizeMod,
                                 Value grainsize, mlir::Type grainsizeType) {
  printGranularityClause<ClauseGrainsizeTypeAttr, ClauseGrainsizeType>(
      p, op, grainsizeMod, grainsize, grainsizeType,
      &stringifyClauseGrainsizeType);
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for num_tasks Clause
//===----------------------------------------------------------------------===//
 
// numtask ::= `num_tasks` `(` [strict ':'] num-tasks `)`
static ParseResult
parseNumTasksClause(OpAsmParser &parser, ClauseNumTasksTypeAttr &numTasksMod,
                    std::optional<OpAsmParser::UnresolvedOperand> &numTasks,
                    Type &numTasksType) {
  return parseGranularityClause<ClauseNumTasksTypeAttr, ClauseNumTasksType>(
      parser, numTasksMod, numTasks, numTasksType, &symbolizeClauseNumTasksType,
      "num_tasks");
}
 
static void printNumTasksClause(OpAsmPrinter &p, Operation *op,
                                ClauseNumTasksTypeAttr numTasksMod,
                                Value numTasks, mlir::Type numTasksType) {
  printGranularityClause<ClauseNumTasksTypeAttr, ClauseNumTasksType>(
      p, op, numTasksMod, numTasks, numTasksType, &stringifyClauseNumTasksType);
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for Heap Alloc Clause
//===----------------------------------------------------------------------===//
 
/// operation ::= $in_type ( `(` $typeparams `)` )? ( `,` $shape )?
static ParseResult parseHeapAllocClause(
    OpAsmParser &parser, TypeAttr &inTypeAttr,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &typeparams,
    SmallVectorImpl<Type> &typeparamsTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &shape,
    SmallVectorImpl<Type> &shapeTypes) {
  mlir::Type inType;
  if (parser.parseType(inType))
    return mlir::failure();
  inTypeAttr = TypeAttr::get(inType);
 
  if (!parser.parseOptionalLParen()) {
    // parse the LEN params of the derived type. (<params> : <types>)
    if (parser.parseOperandList(typeparams, OpAsmParser::Delimiter::None) ||
        parser.parseColonTypeList(typeparamsTypes) || parser.parseRParen())
      return failure();
  }
 
  if (!parser.parseOptionalComma()) {
    // parse size to scale by, vector of n dimensions of type index
    if (parser.parseOperandList(shape, OpAsmParser::Delimiter::None))
      return failure();
 
    // TODO: This overrides the actual types of the operands, which might cause
    // issues when they don't match. At the moment this is done in place of
    // making the corresponding operand type `Variadic<Index>` because index
    // types are lowered to I64 prior to LLVM IR translation.
    shapeTypes.append(shape.size(), IndexType::get(parser.getContext()));
  }
 
  return success();
}
 
static void printHeapAllocClause(OpAsmPrinter &p, Operation *op,
                                 TypeAttr inType, ValueRange typeparams,
                                 TypeRange typeparamsTypes, ValueRange shape,
                                 TypeRange shapeTypes) {
  p << inType;
  if (!typeparams.empty()) {
    p << '(' << typeparams << " : " << typeparamsTypes << ')';
  }
  for (auto sh : shape) {
    p << ", ";
    p.printOperand(sh);
  }
}
 
//===----------------------------------------------------------------------===//
// Parser, printer and verify for dyn_groupprivate Clause
//===----------------------------------------------------------------------===//
 
static LogicalResult
verifyDynGroupprivateClause(Operation *op, AccessGroupModifierAttr accessGroup,
                            FallbackModifierAttr fallback,
                            Value dynGroupprivateSize) {
  if (!dynGroupprivateSize && (accessGroup || fallback))
    return op->emitOpError("dyn_groupprivate modifiers require a size operand");
 
  return success();
}
 
static ParseResult parseDynGroupprivateClause(
    OpAsmParser &parser, AccessGroupModifierAttr &accessGroupAttr,
    FallbackModifierAttr &fallbackAttr,
    std::optional<OpAsmParser::UnresolvedOperand> &dynGroupprivateSize,
    Type &sizeType) {
 
  bool parsedAccessGroup = false;
  bool parsedFallback = false;
  bool parsedSize = false;
 
  return parser.parseCommaSeparatedList([&]() -> ParseResult {
    // Parse AccessGroupModifier.
    if (succeeded(parser.parseOptionalKeyword("cgroup"))) {
      if (parsedAccessGroup)
        return parser.emitError(parser.getCurrentLocation(),
                                "duplicate access group modifier");
      accessGroupAttr = AccessGroupModifierAttr::get(
          parser.getContext(), AccessGroupModifier::cgroup);
      parsedAccessGroup = true;
      return success();
    }
    // Parse FallbackModifier.
    if (succeeded(parser.parseOptionalKeyword("fallback"))) {
      if (parsedFallback)
        return parser.emitError(parser.getCurrentLocation(),
                                "duplicate fallback modifier");
      if (parser.parseLParen())
        return parser.emitError(parser.getCurrentLocation(),
                                "expected '(' after 'fallback'");
      llvm::StringRef fbKind;
      if (parser.parseKeyword(&fbKind))
        return parser.emitError(
            parser.getCurrentLocation(),
            "expected fallback modifier (abort/null/default_mem)");
      std::optional<FallbackModifier> fbEnum;
      if (fbKind == "abort")
        fbEnum = FallbackModifier::abort;
      else if (fbKind == "null")
        fbEnum = FallbackModifier::null;
      else if (fbKind == "default_mem")
        fbEnum = FallbackModifier::default_mem;
      else
        return parser.emitError(parser.getCurrentLocation(),
                                "invalid fallback modifier '" + fbKind + "'");
      fallbackAttr = FallbackModifierAttr::get(parser.getContext(), *fbEnum);
      if (parser.parseRParen())
        return parser.emitError(parser.getCurrentLocation(),
                                "expected ')' after fallback modifier");
      parsedFallback = true;
      return success();
    }
    // Parse size operand.
    OpAsmParser::UnresolvedOperand operand;
    if (succeeded(parser.parseOperand(operand))) {
      if (parsedSize)
        return parser.emitError(parser.getCurrentLocation(),
                                "duplicate size operand");
      dynGroupprivateSize = operand;
      parsedSize = true;
      if (failed(parser.parseColon()) || failed(parser.parseType(sizeType)))
        return parser.emitError(parser.getCurrentLocation(),
                                "expected ':' and type after size operand");
      return success();
    }
    return parser.emitError(parser.getCurrentLocation(),
                            "expected dyn_groupprivate_size operand");
  });
}
 
static void printDynGroupprivateClause(OpAsmPrinter &printer, Operation *op,
                                       AccessGroupModifierAttr modifierFirst,
                                       FallbackModifierAttr modifierSecond,
                                       Value dynGroupprivateSize,
                                       Type sizeType) {
 
  bool needsComma = false;
 
  if (modifierFirst) {
    printer << modifierFirst.getValue();
    needsComma = true;
  }
 
  if (modifierSecond) {
    if (needsComma)
      printer << ", ";
    printer << "fallback(";
    printer << modifierSecond.getValue();
    printer << ")";
    needsComma = true;
  }
 
  if (dynGroupprivateSize) {
    if (needsComma)
      printer << ", ";
    printer << dynGroupprivateSize << " : " << sizeType;
  }
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for in_reduction Clause
//===----------------------------------------------------------------------===//
 
/// Parses an `in_reduction` clause for an operation that does not give its
/// list items entry block arguments (e.g. `omp.target`). The expected format is
/// a comma-separated list of `[byref] @sym %var` followed by `: types`.
static ParseResult parseInReductionClause(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
    SmallVectorImpl<Type> &inReductionTypes,
    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms) {
  SmallVector<SymbolRefAttr> symbolVec;
  SmallVector<bool> isByRefVec;
 
  if (parser.parseCommaSeparatedList([&]() {
        isByRefVec.push_back(parser.parseOptionalKeyword("byref").succeeded());
        if (parser.parseAttribute(symbolVec.emplace_back()) ||
            parser.parseOperand(inReductionVars.emplace_back()))
          return failure();
        return success();
      }))
    return failure();
 
  if (parser.parseColon())
    return failure();
 
  if (parser.parseCommaSeparatedList(
          [&]() { return parser.parseType(inReductionTypes.emplace_back()); }))
    return failure();
 
  if (inReductionVars.size() != inReductionTypes.size())
    return failure();
 
  inReductionByref = makeDenseBoolArrayAttr(parser.getContext(), isByRefVec);
  SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());
  inReductionSyms = ArrayAttr::get(parser.getContext(), symbolAttrs);
  return success();
}
 
/// Prints an `in_reduction` clause for an operation that does not give its list
/// items entry block arguments (e.g. `omp.target`). Mirrors
/// `parseInReductionClause`.
static void printInReductionClause(OpAsmPrinter &p, Operation *op,
                                   ValueRange inReductionVars,
                                   TypeRange inReductionTypes,
                                   DenseBoolArrayAttr inReductionByref,
                                   ArrayAttr inReductionSyms) {
  MLIRContext *ctx = op->getContext();
 
  ArrayAttr syms = inReductionSyms;
  if (!syms) {
    SmallVector<Attribute> values(inReductionVars.size(), nullptr);
    syms = ArrayAttr::get(ctx, values);
  }
 
  DenseBoolArrayAttr byref = inReductionByref;
  if (!byref) {
    SmallVector<bool> values(inReductionVars.size(), false);
    byref = DenseBoolArrayAttr::get(ctx, values);
  }
 
  llvm::interleaveComma(
      llvm::zip_equal(inReductionVars, syms.getValue(), byref.asArrayRef()), p,
      [&p](auto t) {
        auto [var, sym, isByRef] = t;
        if (isByRef)
          p << "byref ";
        if (sym)
          p << sym << " ";
        p << var;
      });
  p << " : ";
  llvm::interleaveComma(inReductionTypes, p);
}
 
//===----------------------------------------------------------------------===//
// Parsers for operations including clauses that define entry block arguments.
//===----------------------------------------------------------------------===//
 
namespace {
struct MapParseArgs {
  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
  SmallVectorImpl<Type> &types;
  MapParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
               SmallVectorImpl<Type> &types)
      : vars(vars), types(types) {}
};
struct PrivateParseArgs {
  llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
  llvm::SmallVectorImpl<Type> &types;
  ArrayAttr &syms;
  UnitAttr &needsBarrier;
  DenseI64ArrayAttr *mapIndices;
  PrivateParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
                   SmallVectorImpl<Type> &types, ArrayAttr &syms,
                   UnitAttr &needsBarrier,
                   DenseI64ArrayAttr *mapIndices = nullptr)
      : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
        mapIndices(mapIndices) {}
};
 
struct LinearParseArgs {
  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
  SmallVectorImpl<Type> &types;
  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars;
  SmallVectorImpl<Type> &stepTypes;
  ArrayAttr &varTypes;
  ArrayAttr &modifiers;
  ArrayAttr &syms;
  LinearParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
                  SmallVectorImpl<Type> &types,
                  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &stepVars,
                  SmallVectorImpl<Type> &stepTypes, ArrayAttr &varTypes,
                  ArrayAttr &modifiers, ArrayAttr &syms)
      : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
        varTypes(varTypes), modifiers(modifiers), syms(syms) {}
};
 
struct ReductionParseArgs {
  SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars;
  SmallVectorImpl<Type> &types;
  DenseBoolArrayAttr &byref;
  ArrayAttr &syms;
  ReductionModifierAttr *modifier;
  ReductionParseArgs(SmallVectorImpl<OpAsmParser::UnresolvedOperand> &vars,
                     SmallVectorImpl<Type> &types, DenseBoolArrayAttr &byref,
                     ArrayAttr &syms, ReductionModifierAttr *mod = nullptr)
      : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
};
 
struct AllRegionParseArgs {
  std::optional<MapParseArgs> hasDeviceAddrArgs;
  std::optional<MapParseArgs> hostEvalArgs;
  std::optional<ReductionParseArgs> inReductionArgs;
  std::optional<LinearParseArgs> linearArgs;
  std::optional<MapParseArgs> mapArgs;
  std::optional<PrivateParseArgs> privateArgs;
  std::optional<ReductionParseArgs> reductionArgs;
  std::optional<ReductionParseArgs> taskReductionArgs;
  std::optional<MapParseArgs> useDeviceAddrArgs;
  std::optional<MapParseArgs> useDevicePtrArgs;
};
} // namespace
 
static inline constexpr StringRef getPrivateNeedsBarrierSpelling() {
  return "private_barrier";
}
 
static ParseResult parseClauseWithRegionArgs(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands,
    SmallVectorImpl<Type> &types,
    SmallVectorImpl<OpAsmParser::Argument> &regionPrivateArgs,
    ArrayAttr *symbols = nullptr, DenseI64ArrayAttr *mapIndices = nullptr,
    DenseBoolArrayAttr *byref = nullptr,
    ReductionModifierAttr *modifier = nullptr,
    UnitAttr *needsBarrier = nullptr) {
  SmallVector<SymbolRefAttr> symbolVec;
  SmallVector<int64_t> mapIndicesVec;
  SmallVector<bool> isByRefVec;
  unsigned regionArgOffset = regionPrivateArgs.size();
 
  if (parser.parseLParen())
    return failure();
 
  if (modifier && succeeded(parser.parseOptionalKeyword("mod"))) {
    StringRef enumStr;
    if (parser.parseColon() || parser.parseKeyword(&enumStr) ||
        parser.parseComma())
      return failure();
    std::optional<ReductionModifier> enumValue =
        symbolizeReductionModifier(enumStr);
    if (!enumValue.has_value())
      return failure();
    *modifier = ReductionModifierAttr::get(parser.getContext(), *enumValue);
    if (!*modifier)
      return failure();
  }
 
  if (parser.parseCommaSeparatedList([&]() {
        if (byref)
          isByRefVec.push_back(
              parser.parseOptionalKeyword("byref").succeeded());
 
        if (symbols && parser.parseAttribute(symbolVec.emplace_back()))
          return failure();
 
        if (parser.parseOperand(operands.emplace_back()) ||
            parser.parseArrow() ||
            parser.parseArgument(regionPrivateArgs.emplace_back()))
          return failure();
 
        if (mapIndices) {
          if (parser.parseOptionalLSquare().succeeded()) {
            if (parser.parseKeyword("map_idx") || parser.parseEqual() ||
                parser.parseInteger(mapIndicesVec.emplace_back()) ||
                parser.parseRSquare())
              return failure();
          } else {
            mapIndicesVec.push_back(-1);
          }
        }
 
        return success();
      }))
    return failure();
 
  if (parser.parseColon())
    return failure();
 
  if (parser.parseCommaSeparatedList([&]() {
        if (parser.parseType(types.emplace_back()))
          return failure();
 
        return success();
      }))
    return failure();
 
  if (operands.size() != types.size())
    return failure();
 
  if (parser.parseRParen())
    return failure();
 
  if (needsBarrier) {
    if (parser.parseOptionalKeyword(getPrivateNeedsBarrierSpelling())
            .succeeded())
      *needsBarrier = mlir::UnitAttr::get(parser.getContext());
  }
 
  auto *argsBegin = regionPrivateArgs.begin();
  MutableArrayRef argsSubrange(argsBegin + regionArgOffset,
                               argsBegin + regionArgOffset + types.size());
  for (auto [prv, type] : llvm::zip_equal(argsSubrange, types)) {
    prv.type = type;
  }
 
  if (symbols) {
    SmallVector<Attribute> symbolAttrs(symbolVec.begin(), symbolVec.end());
    *symbols = ArrayAttr::get(parser.getContext(), symbolAttrs);
  }
 
  if (!mapIndicesVec.empty())
    *mapIndices =
        mlir::DenseI64ArrayAttr::get(parser.getContext(), mapIndicesVec);
 
  if (byref)
    *byref = makeDenseBoolArrayAttr(parser.getContext(), isByRefVec);
 
  return success();
}
 
static ParseResult parseBlockArgClause(
    OpAsmParser &parser,
    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
    StringRef keyword, std::optional<MapParseArgs> mapArgs) {
  if (succeeded(parser.parseOptionalKeyword(keyword))) {
    if (!mapArgs)
      return failure();
 
    if (failed(parseClauseWithRegionArgs(parser, mapArgs->vars, mapArgs->types,
                                         entryBlockArgs)))
      return failure();
  }
  return success();
}
 
static ParseResult parseBlockArgClause(
    OpAsmParser &parser,
    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
    StringRef keyword, std::optional<PrivateParseArgs> privateArgs) {
  if (succeeded(parser.parseOptionalKeyword(keyword))) {
    if (!privateArgs)
      return failure();
 
    if (failed(parseClauseWithRegionArgs(
            parser, privateArgs->vars, privateArgs->types, entryBlockArgs,
            &privateArgs->syms, privateArgs->mapIndices, /*byref=*/nullptr,
            /*modifier=*/nullptr, &privateArgs->needsBarrier)))
      return failure();
  }
  return success();
}
 
static ParseResult parseBlockArgClause(
    OpAsmParser &parser,
    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
    StringRef keyword, std::optional<ReductionParseArgs> reductionArgs) {
  if (succeeded(parser.parseOptionalKeyword(keyword))) {
    if (!reductionArgs)
      return failure();
    if (failed(parseClauseWithRegionArgs(
            parser, reductionArgs->vars, reductionArgs->types, entryBlockArgs,
            &reductionArgs->syms, /*mapIndices=*/nullptr, &reductionArgs->byref,
            reductionArgs->modifier)))
      return failure();
  }
  return success();
}
 
/// linear-entry := [modifier `(`] symbol ssa-id `->` ssa-id `:` type `=`
///                 ssa-id `:` type [`)`]
static ParseResult parseBlockArgClause(
    OpAsmParser &parser,
    llvm::SmallVectorImpl<OpAsmParser::Argument> &entryBlockArgs,
    StringRef keyword, std::optional<LinearParseArgs> linearArgs) {
  if (!linearArgs || failed(parser.parseOptionalKeyword(keyword)))
    return success();
 
  MLIRContext *ctx = parser.getContext();
  SmallVector<Attribute> syms, modifiers;
  if (parser.parseLParen() ||
      parser.parseCommaSeparatedList([&]() -> ParseResult {
        std::optional<LinearModifier> modifier;
        if (succeeded(parser.parseOptionalKeyword("val")))
          modifier = LinearModifier::val;
        else if (succeeded(parser.parseOptionalKeyword("ref")))
          modifier = LinearModifier::ref;
        else if (succeeded(parser.parseOptionalKeyword("uval")))
          modifier = LinearModifier::uval;
        if (modifier && parser.parseLParen())
          return failure();
 
        SymbolRefAttr sym;
        OpAsmParser::Argument &arg = entryBlockArgs.emplace_back();
        if (parser.parseAttribute(sym) ||
            parser.parseOperand(linearArgs->vars.emplace_back()) ||
            parser.parseArrow() || parser.parseArgument(arg) ||
            parser.parseColonType(linearArgs->types.emplace_back()) ||
            parser.parseEqual() ||
            parser.parseOperand(linearArgs->stepVars.emplace_back()) ||
            parser.parseColonType(linearArgs->stepTypes.emplace_back()))
          return failure();
        arg.type = linearArgs->types.back();
 
        if (modifier && parser.parseRParen())
          return failure();
 
        syms.push_back(sym);
        modifiers.push_back(
            modifier ? Attribute(LinearModifierAttr::get(ctx, *modifier))
                     : Attribute(UnitAttr::get(ctx)));
        return success();
      }) ||
      parser.parseRParen())
    return failure();
 
  linearArgs->syms = ArrayAttr::get(ctx, syms);
  linearArgs->modifiers = ArrayAttr::get(ctx, modifiers);
 
  if (succeeded(parser.parseOptionalKeyword("linear_var_types")) &&
      (parser.parseLParen() || parser.parseAttribute(linearArgs->varTypes) ||
       parser.parseRParen()))
    return failure();
  return success();
}
 
static ParseResult parseBlockArgRegion(OpAsmParser &parser, Region &region,
                                       AllRegionParseArgs args) {
  llvm::SmallVector<OpAsmParser::Argument> entryBlockArgs;
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "has_device_addr",
                                 args.hasDeviceAddrArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `has_device_addr` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "host_eval",
                                 args.hostEvalArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `host_eval` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "in_reduction",
                                 args.inReductionArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `in_reduction` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "linear",
                                 args.linearArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `linear` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "map_entries",
                                 args.mapArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `map_entries` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "private",
                                 args.privateArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `private` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "reduction",
                                 args.reductionArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `reduction` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "task_reduction",
                                 args.taskReductionArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `task_reduction` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "use_device_addr",
                                 args.useDeviceAddrArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `use_device_addr` format";
 
  if (failed(parseBlockArgClause(parser, entryBlockArgs, "use_device_ptr",
                                 args.useDevicePtrArgs)))
    return parser.emitError(parser.getCurrentLocation())
           << "invalid `use_device_addr` format";
 
  return parser.parseRegion(region, entryBlockArgs);
}
 
// These parseXyz functions correspond to the custom<Xyz> definitions
// in the .td file(s).
static ParseResult parseTargetOpRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hasDeviceAddrVars,
    SmallVectorImpl<Type> &hasDeviceAddrTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &hostEvalVars,
    SmallVectorImpl<Type> &hostEvalTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &mapVars,
    SmallVectorImpl<Type> &mapTypes,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier, DenseI64ArrayAttr &privateMaps) {
  AllRegionParseArgs args;
  args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
  args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
  args.mapArgs.emplace(mapVars, mapTypes);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier, &privateMaps);
  return parseBlockArgRegion(parser, region, args);
}
 
static ParseResult parseInReductionPrivateRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
    SmallVectorImpl<Type> &inReductionTypes,
    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier) {
  AllRegionParseArgs args;
  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                               inReductionByref, inReductionSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  return parseBlockArgRegion(parser, region, args);
}
 
static ParseResult parseInReductionPrivateReductionRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &inReductionVars,
    SmallVectorImpl<Type> &inReductionTypes,
    DenseBoolArrayAttr &inReductionByref, ArrayAttr &inReductionSyms,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
    SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
    ArrayAttr &reductionSyms) {
  AllRegionParseArgs args;
  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                               inReductionByref, inReductionSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, &reductionMod);
  return parseBlockArgRegion(parser, region, args);
}
 
static ParseResult parsePrivateRegion(
    OpAsmParser &parser, Region &region,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier) {
  AllRegionParseArgs args;
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  return parseBlockArgRegion(parser, region, args);
}
 
static ParseResult parsePrivateReductionRegion(
    OpAsmParser &parser, Region &region,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
    SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
    ArrayAttr &reductionSyms) {
  AllRegionParseArgs args;
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, &reductionMod);
  return parseBlockArgRegion(parser, region, args);
}
 
static ParseResult parseLinearPrivateReductionRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearVars,
    SmallVectorImpl<Type> &linearTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &linearStepVars,
    SmallVectorImpl<Type> &linearStepTypes, ArrayAttr &linearVarTypes,
    ArrayAttr &linearModifiers, ArrayAttr &linearSyms,
    llvm::SmallVectorImpl<OpAsmParser::UnresolvedOperand> &privateVars,
    llvm::SmallVectorImpl<Type> &privateTypes, ArrayAttr &privateSyms,
    UnitAttr &privateNeedsBarrier, ReductionModifierAttr &reductionMod,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &reductionVars,
    SmallVectorImpl<Type> &reductionTypes, DenseBoolArrayAttr &reductionByref,
    ArrayAttr &reductionSyms) {
  AllRegionParseArgs args;
  args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
                          linearStepTypes, linearVarTypes, linearModifiers,
                          linearSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, &reductionMod);
  return parseBlockArgRegion(parser, region, args);
}
 
static ParseResult parseTaskReductionRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &taskReductionVars,
    SmallVectorImpl<Type> &taskReductionTypes,
    DenseBoolArrayAttr &taskReductionByref, ArrayAttr &taskReductionSyms) {
  AllRegionParseArgs args;
  args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,
                                 taskReductionByref, taskReductionSyms);
  return parseBlockArgRegion(parser, region, args);
}
 
static ParseResult parseUseDeviceAddrUseDevicePtrRegion(
    OpAsmParser &parser, Region &region,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDeviceAddrVars,
    SmallVectorImpl<Type> &useDeviceAddrTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &useDevicePtrVars,
    SmallVectorImpl<Type> &useDevicePtrTypes) {
  AllRegionParseArgs args;
  args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);
  args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);
  return parseBlockArgRegion(parser, region, args);
}
 
//===----------------------------------------------------------------------===//
// Printers for operations including clauses that define entry block arguments.
//===----------------------------------------------------------------------===//
 
namespace {
struct MapPrintArgs {
  ValueRange vars;
  TypeRange types;
  MapPrintArgs(ValueRange vars, TypeRange types) : vars(vars), types(types) {}
};
struct PrivatePrintArgs {
  ValueRange vars;
  TypeRange types;
  ArrayAttr syms;
  UnitAttr needsBarrier;
  DenseI64ArrayAttr mapIndices;
  PrivatePrintArgs(ValueRange vars, TypeRange types, ArrayAttr syms,
                   UnitAttr needsBarrier, DenseI64ArrayAttr mapIndices)
      : vars(vars), types(types), syms(syms), needsBarrier(needsBarrier),
        mapIndices(mapIndices) {}
};
struct LinearPrintArgs {
  ValueRange vars;
  TypeRange types;
  ValueRange stepVars;
  TypeRange stepTypes;
  ArrayAttr varTypes;
  ArrayAttr modifiers;
  ArrayAttr syms;
  LinearPrintArgs(ValueRange vars, TypeRange types, ValueRange stepVars,
                  TypeRange stepTypes, ArrayAttr varTypes, ArrayAttr modifiers,
                  ArrayAttr syms)
      : vars(vars), types(types), stepVars(stepVars), stepTypes(stepTypes),
        varTypes(varTypes), modifiers(modifiers), syms(syms) {}
};
struct ReductionPrintArgs {
  ValueRange vars;
  TypeRange types;
  DenseBoolArrayAttr byref;
  ArrayAttr syms;
  ReductionModifierAttr modifier;
  ReductionPrintArgs(ValueRange vars, TypeRange types, DenseBoolArrayAttr byref,
                     ArrayAttr syms, ReductionModifierAttr mod = nullptr)
      : vars(vars), types(types), byref(byref), syms(syms), modifier(mod) {}
};
struct AllRegionPrintArgs {
  std::optional<MapPrintArgs> hasDeviceAddrArgs;
  std::optional<MapPrintArgs> hostEvalArgs;
  std::optional<ReductionPrintArgs> inReductionArgs;
  std::optional<LinearPrintArgs> linearArgs;
  std::optional<MapPrintArgs> mapArgs;
  std::optional<PrivatePrintArgs> privateArgs;
  std::optional<ReductionPrintArgs> reductionArgs;
  std::optional<ReductionPrintArgs> taskReductionArgs;
  std::optional<MapPrintArgs> useDeviceAddrArgs;
  std::optional<MapPrintArgs> useDevicePtrArgs;
};
} // namespace
 
static void printClauseWithRegionArgs(
    OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
    ValueRange argsSubrange, ValueRange operands, TypeRange types,
    ArrayAttr symbols = nullptr, DenseI64ArrayAttr mapIndices = nullptr,
    DenseBoolArrayAttr byref = nullptr,
    ReductionModifierAttr modifier = nullptr, UnitAttr needsBarrier = nullptr) {
  if (argsSubrange.empty())
    return;
 
  p << clauseName << "(";
 
  if (modifier)
    p << "mod: " << stringifyReductionModifier(modifier.getValue()) << ", ";
 
  if (!symbols) {
    llvm::SmallVector<Attribute> values(operands.size(), nullptr);
    symbols = ArrayAttr::get(ctx, values);
  }
 
  if (!mapIndices) {
    llvm::SmallVector<int64_t> values(operands.size(), -1);
    mapIndices = DenseI64ArrayAttr::get(ctx, values);
  }
 
  if (!byref) {
    mlir::SmallVector<bool> values(operands.size(), false);
    byref = DenseBoolArrayAttr::get(ctx, values);
  }
 
  llvm::interleaveComma(llvm::zip_equal(operands, argsSubrange, symbols,
                                        mapIndices.asArrayRef(),
                                        byref.asArrayRef()),
                        p, [&p](auto t) {
                          auto [op, arg, sym, map, isByRef] = t;
                          if (isByRef)
                            p << "byref ";
                          if (sym)
                            p << sym << " ";
 
                          p << op << " -> " << arg;
 
                          if (map != -1)
                            p << " [map_idx=" << map << "]";
                        });
  p << " : ";
  llvm::interleaveComma(types, p);
  p << ") ";
 
  if (needsBarrier)
    p << getPrivateNeedsBarrierSpelling() << " ";
}
 
static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
                                StringRef clauseName, ValueRange argsSubrange,
                                std::optional<MapPrintArgs> mapArgs) {
  if (mapArgs)
    printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange, mapArgs->vars,
                              mapArgs->types);
}
 
static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
                                StringRef clauseName, ValueRange argsSubrange,
                                std::optional<PrivatePrintArgs> privateArgs) {
  if (privateArgs)
    printClauseWithRegionArgs(
        p, ctx, clauseName, argsSubrange, privateArgs->vars, privateArgs->types,
        privateArgs->syms, privateArgs->mapIndices, /*byref=*/nullptr,
        /*modifier=*/nullptr, privateArgs->needsBarrier);
}
 
static void
printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx, StringRef clauseName,
                    ValueRange argsSubrange,
                    std::optional<ReductionPrintArgs> reductionArgs) {
  if (reductionArgs)
    printClauseWithRegionArgs(p, ctx, clauseName, argsSubrange,
                              reductionArgs->vars, reductionArgs->types,
                              reductionArgs->syms, /*mapIndices=*/nullptr,
                              reductionArgs->byref, reductionArgs->modifier);
}
 
static void printBlockArgClause(OpAsmPrinter &p, MLIRContext *ctx,
                                StringRef clauseName, ValueRange argsSubrange,
                                std::optional<LinearPrintArgs> linearArgs) {
  if (!linearArgs || argsSubrange.empty())
    return;
 
  p << clauseName << "(";
  for (auto [i, var] : llvm::enumerate(linearArgs->vars)) {
    if (i != 0)
      p << ", ";
    auto modifier =
        linearArgs->modifiers
            ? dyn_cast_if_present<LinearModifierAttr>(linearArgs->modifiers[i])
            : nullptr;
    if (modifier)
      p << stringifyLinearModifier(modifier.getValue()) << "(";
    if (linearArgs->syms)
      p << linearArgs->syms[i] << " ";
    p << var << " -> " << argsSubrange[i] << " : " << linearArgs->types[i]
      << " = " << linearArgs->stepVars[i] << " : " << linearArgs->stepTypes[i];
    if (modifier)
      p << ")";
  }
  p << ") ";
 
  if (linearArgs->varTypes)
    p << "linear_var_types(" << linearArgs->varTypes << ") ";
}
 
static void printBlockArgRegion(OpAsmPrinter &p, Operation *op, Region &region,
                                const AllRegionPrintArgs &args) {
  auto iface = llvm::cast<mlir::omp::BlockArgOpenMPOpInterface>(op);
  MLIRContext *ctx = op->getContext();
 
  printBlockArgClause(p, ctx, "has_device_addr",
                      iface.getHasDeviceAddrBlockArgs(),
                      args.hasDeviceAddrArgs);
  printBlockArgClause(p, ctx, "host_eval", iface.getHostEvalBlockArgs(),
                      args.hostEvalArgs);
  printBlockArgClause(p, ctx, "in_reduction", iface.getInReductionBlockArgs(),
                      args.inReductionArgs);
  printBlockArgClause(p, ctx, "linear", iface.getLinearBlockArgs(),
                      args.linearArgs);
  printBlockArgClause(p, ctx, "map_entries", iface.getMapBlockArgs(),
                      args.mapArgs);
  printBlockArgClause(p, ctx, "private", iface.getPrivateBlockArgs(),
                      args.privateArgs);
  printBlockArgClause(p, ctx, "reduction", iface.getReductionBlockArgs(),
                      args.reductionArgs);
  printBlockArgClause(p, ctx, "task_reduction",
                      iface.getTaskReductionBlockArgs(),
                      args.taskReductionArgs);
  printBlockArgClause(p, ctx, "use_device_addr",
                      iface.getUseDeviceAddrBlockArgs(),
                      args.useDeviceAddrArgs);
  printBlockArgClause(p, ctx, "use_device_ptr",
                      iface.getUseDevicePtrBlockArgs(), args.useDevicePtrArgs);
 
  p.printRegion(region, /*printEntryBlockArgs=*/false);
}
 
// These parseXyz functions correspond to the custom<Xyz> definitions
// in the .td file(s).
static void printTargetOpRegion(OpAsmPrinter &p, Operation *op, Region &region,
                                ValueRange hasDeviceAddrVars,
                                TypeRange hasDeviceAddrTypes,
                                ValueRange hostEvalVars,
                                TypeRange hostEvalTypes, ValueRange mapVars,
                                TypeRange mapTypes, ValueRange privateVars,
                                TypeRange privateTypes, ArrayAttr privateSyms,
                                UnitAttr privateNeedsBarrier,
                                DenseI64ArrayAttr privateMaps) {
  AllRegionPrintArgs args;
  args.hasDeviceAddrArgs.emplace(hasDeviceAddrVars, hasDeviceAddrTypes);
  args.hostEvalArgs.emplace(hostEvalVars, hostEvalTypes);
  args.mapArgs.emplace(mapVars, mapTypes);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier, privateMaps);
  printBlockArgRegion(p, op, region, args);
}
 
static void printInReductionPrivateRegion(
    OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
    TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
    ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
    ArrayAttr privateSyms, UnitAttr privateNeedsBarrier) {
  AllRegionPrintArgs args;
  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                               inReductionByref, inReductionSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  printBlockArgRegion(p, op, region, args);
}
 
static void printInReductionPrivateReductionRegion(
    OpAsmPrinter &p, Operation *op, Region &region, ValueRange inReductionVars,
    TypeRange inReductionTypes, DenseBoolArrayAttr inReductionByref,
    ArrayAttr inReductionSyms, ValueRange privateVars, TypeRange privateTypes,
    ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
    ReductionModifierAttr reductionMod, ValueRange reductionVars,
    TypeRange reductionTypes, DenseBoolArrayAttr reductionByref,
    ArrayAttr reductionSyms) {
  AllRegionPrintArgs args;
  args.inReductionArgs.emplace(inReductionVars, inReductionTypes,
                               inReductionByref, inReductionSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, reductionMod);
  printBlockArgRegion(p, op, region, args);
}
 
static void printPrivateRegion(OpAsmPrinter &p, Operation *op, Region &region,
                               ValueRange privateVars, TypeRange privateTypes,
                               ArrayAttr privateSyms,
                               UnitAttr privateNeedsBarrier) {
  AllRegionPrintArgs args;
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  printBlockArgRegion(p, op, region, args);
}
 
static void printPrivateReductionRegion(
    OpAsmPrinter &p, Operation *op, Region &region, ValueRange privateVars,
    TypeRange privateTypes, ArrayAttr privateSyms, UnitAttr privateNeedsBarrier,
    ReductionModifierAttr reductionMod, ValueRange reductionVars,
    TypeRange reductionTypes, DenseBoolArrayAttr reductionByref,
    ArrayAttr reductionSyms) {
  AllRegionPrintArgs args;
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, reductionMod);
  printBlockArgRegion(p, op, region, args);
}
 
static void printLinearPrivateReductionRegion(
    OpAsmPrinter &p, Operation *op, Region &region, ValueRange linearVars,
    TypeRange linearTypes, ValueRange linearStepVars, TypeRange linearStepTypes,
    ArrayAttr linearVarTypes, ArrayAttr linearModifiers, ArrayAttr linearSyms,
    ValueRange privateVars, TypeRange privateTypes, ArrayAttr privateSyms,
    UnitAttr privateNeedsBarrier, ReductionModifierAttr reductionMod,
    ValueRange reductionVars, TypeRange reductionTypes,
    DenseBoolArrayAttr reductionByref, ArrayAttr reductionSyms) {
  AllRegionPrintArgs args;
  args.linearArgs.emplace(linearVars, linearTypes, linearStepVars,
                          linearStepTypes, linearVarTypes, linearModifiers,
                          linearSyms);
  args.privateArgs.emplace(privateVars, privateTypes, privateSyms,
                           privateNeedsBarrier,
                           /*mapIndices=*/nullptr);
  args.reductionArgs.emplace(reductionVars, reductionTypes, reductionByref,
                             reductionSyms, reductionMod);
  printBlockArgRegion(p, op, region, args);
}
 
static void printTaskReductionRegion(OpAsmPrinter &p, Operation *op,
                                     Region &region,
                                     ValueRange taskReductionVars,
                                     TypeRange taskReductionTypes,
                                     DenseBoolArrayAttr taskReductionByref,
                                     ArrayAttr taskReductionSyms) {
  AllRegionPrintArgs args;
  args.taskReductionArgs.emplace(taskReductionVars, taskReductionTypes,
                                 taskReductionByref, taskReductionSyms);
  printBlockArgRegion(p, op, region, args);
}
 
static void printUseDeviceAddrUseDevicePtrRegion(OpAsmPrinter &p, Operation *op,
                                                 Region &region,
                                                 ValueRange useDeviceAddrVars,
                                                 TypeRange useDeviceAddrTypes,
                                                 ValueRange useDevicePtrVars,
                                                 TypeRange useDevicePtrTypes) {
  AllRegionPrintArgs args;
  args.useDeviceAddrArgs.emplace(useDeviceAddrVars, useDeviceAddrTypes);
  args.useDevicePtrArgs.emplace(useDevicePtrVars, useDevicePtrTypes);
  printBlockArgRegion(p, op, region, args);
}
 
template <typename ParsePrefixFn>
static ParseResult parseSplitIteratedList(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iteratedVars,
    SmallVectorImpl<Type> &iteratedTypes,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &plainVars,
    SmallVectorImpl<Type> &plainTypes, ParsePrefixFn &&parsePrefix) {
 
  return parser.parseCommaSeparatedList([&]() -> ParseResult {
    if (failed(parsePrefix()))
      return failure();
 
    OpAsmParser::UnresolvedOperand v;
    Type ty;
    if (parser.parseOperand(v) || parser.parseColonType(ty))
      return failure();
 
    if (llvm::isa<mlir::omp::IteratedType>(ty)) {
      iteratedVars.push_back(v);
      iteratedTypes.push_back(ty);
    } else {
      plainVars.push_back(v);
      plainTypes.push_back(ty);
    }
    return success();
  });
}
 
template <typename PrintPrefixFn>
static void printSplitIteratedList(OpAsmPrinter &p, ValueRange iteratedVars,
                                   TypeRange iteratedTypes,
                                   ValueRange plainVars, TypeRange plainTypes,
                                   PrintPrefixFn &&printPrefixForPlain,
                                   PrintPrefixFn &&printPrefixForIterated) {
 
  bool first = true;
  auto emit = [&](Value v, Type t, auto &&printPrefix) {
    if (!first)
      p << ", ";
    printPrefix(v, t);
    p << v << " : " << t;
    first = false;
  };
 
  for (unsigned i = 0; i < iteratedVars.size(); ++i)
    emit(iteratedVars[i], iteratedTypes[i], printPrefixForIterated);
  for (unsigned i = 0; i < plainVars.size(); ++i)
    emit(plainVars[i], plainTypes[i], printPrefixForPlain);
}
 
/// Verifies Reduction Clause
static LogicalResult
verifyReductionVarList(Operation *op, std::optional<ArrayAttr> reductionSyms,
                       OperandRange reductionVars,
                       std::optional<ArrayRef<bool>> reductionByref) {
  if (!reductionVars.empty()) {
    if (!reductionSyms || reductionSyms->size() != reductionVars.size())
      return op->emitOpError()
             << "expected as many reduction symbol references "
                "as reduction variables";
    if (reductionByref && reductionByref->size() != reductionVars.size())
      return op->emitError() << "expected as many reduction variable by "
                                "reference attributes as reduction variables";
  } else {
    if (reductionSyms)
      return op->emitOpError() << "unexpected reduction symbol references";
    return success();
  }
 
  // TODO: The followings should be done in
  // SymbolUserOpInterface::verifySymbolUses.
  DenseSet<Value> accumulators;
  for (auto args : llvm::zip(reductionVars, *reductionSyms)) {
    Value accum = std::get<0>(args);
 
    if (!accumulators.insert(accum).second)
      return op->emitOpError() << "accumulator variable used more than once";
 
    Type varType = accum.getType();
    auto symbolRef = llvm::cast<SymbolRefAttr>(std::get<1>(args));
    auto decl =
        SymbolTable::lookupNearestSymbolFrom<DeclareReductionOp>(op, symbolRef);
    if (!decl)
      return op->emitOpError() << "expected symbol reference " << symbolRef
                               << " to point to a reduction declaration";
 
    if (decl.getAccumulatorType() && decl.getAccumulatorType() != varType)
      return op->emitOpError()
             << "expected accumulator (" << varType
             << ") to be the same type as reduction declaration ("
             << decl.getAccumulatorType() << ")";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Parser, printer and verifier for Copyprivate
//===----------------------------------------------------------------------===//
 
/// copyprivate-entry-list ::= copyprivate-entry
///                          | copyprivate-entry-list `,` copyprivate-entry
/// copyprivate-entry ::= ssa-id `->` symbol-ref `:` type
static ParseResult parseCopyprivate(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &copyprivateVars,
    SmallVectorImpl<Type> &copyprivateTypes, ArrayAttr &copyprivateSyms) {
  SmallVector<SymbolRefAttr> symsVec;
  if (failed(parser.parseCommaSeparatedList([&]() {
        if (parser.parseOperand(copyprivateVars.emplace_back()) ||
            parser.parseArrow() ||
            parser.parseAttribute(symsVec.emplace_back()) ||
            parser.parseColonType(copyprivateTypes.emplace_back()))
          return failure();
        return success();
      })))
    return failure();
  SmallVector<Attribute> syms(symsVec.begin(), symsVec.end());
  copyprivateSyms = ArrayAttr::get(parser.getContext(), syms);
  return success();
}
 
/// Print Copyprivate clause
static void printCopyprivate(OpAsmPrinter &p, Operation *op,
                             OperandRange copyprivateVars,
                             TypeRange copyprivateTypes,
                             std::optional<ArrayAttr> copyprivateSyms) {
  if (!copyprivateSyms.has_value())
    return;
  llvm::interleaveComma(
      llvm::zip(copyprivateVars, *copyprivateSyms, copyprivateTypes), p,
      [&](const auto &args) {
        p << std::get<0>(args) << " -> " << std::get<1>(args) << " : "
          << std::get<2>(args);
      });
}
 
/// Verifies CopyPrivate Clause
static LogicalResult
verifyCopyprivateVarList(Operation *op, OperandRange copyprivateVars,
                         std::optional<ArrayAttr> copyprivateSyms) {
  size_t copyprivateSymsSize =
      copyprivateSyms.has_value() ? copyprivateSyms->size() : 0;
  if (copyprivateSymsSize != copyprivateVars.size())
    return op->emitOpError() << "inconsistent number of copyprivate vars (= "
                             << copyprivateVars.size()
                             << ") and functions (= " << copyprivateSymsSize
                             << "), both must be equal";
  if (!copyprivateSyms.has_value())
    return success();
 
  for (auto copyprivateVarAndSym :
       llvm::zip(copyprivateVars, *copyprivateSyms)) {
    auto symbolRef =
        llvm::cast<SymbolRefAttr>(std::get<1>(copyprivateVarAndSym));
    std::optional<std::variant<mlir::func::FuncOp, mlir::LLVM::LLVMFuncOp>>
        funcOp;
    if (mlir::func::FuncOp mlirFuncOp =
            SymbolTable::lookupNearestSymbolFrom<mlir::func::FuncOp>(op,
                                                                     symbolRef))
      funcOp = mlirFuncOp;
    else if (mlir::LLVM::LLVMFuncOp llvmFuncOp =
                 SymbolTable::lookupNearestSymbolFrom<mlir::LLVM::LLVMFuncOp>(
                     op, symbolRef))
      funcOp = llvmFuncOp;
 
    auto getNumArguments = [&] {
      return std::visit([](auto &f) { return f.getNumArguments(); }, *funcOp);
    };
 
    auto getArgumentType = [&](unsigned i) {
      return std::visit([i](auto &f) { return f.getArgumentTypes()[i]; },
                        *funcOp);
    };
 
    if (!funcOp)
      return op->emitOpError() << "expected symbol reference " << symbolRef
                               << " to point to a copy function";
 
    if (getNumArguments() != 2)
      return op->emitOpError()
             << "expected copy function " << symbolRef << " to have 2 operands";
 
    Type argTy = getArgumentType(0);
    if (argTy != getArgumentType(1))
      return op->emitOpError() << "expected copy function " << symbolRef
                               << " arguments to have the same type";
 
    Type varType = std::get<0>(copyprivateVarAndSym).getType();
    if (argTy != varType)
      return op->emitOpError()
             << "expected copy function arguments' type (" << argTy
             << ") to be the same as copyprivate variable's type (" << varType
             << ")";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Parser, printer and verifier for DependVarList
//===----------------------------------------------------------------------===//
 
/// depend-entry-list ::= depend-entry
///                     | depend-entry-list `,` depend-entry
/// depend-entry ::= depend-kind `->` ssa-id `:` type
///                | depend-kind `->` ssa-id `:` iterated-type
static ParseResult parseDependVarList(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &dependVars,
    SmallVectorImpl<Type> &dependTypes, ArrayAttr &dependKinds,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iteratedVars,
    SmallVectorImpl<Type> &iteratedTypes, ArrayAttr &iteratedKinds) {
  SmallVector<ClauseTaskDependAttr> kindsVec;
  SmallVector<ClauseTaskDependAttr> iterKindsVec;
  if (failed(parser.parseCommaSeparatedList([&]() {
        StringRef keyword;
        OpAsmParser::UnresolvedOperand operand;
        Type ty;
        if (parser.parseKeyword(&keyword) || parser.parseArrow() ||
            parser.parseOperand(operand) || parser.parseColonType(ty))
          return failure();
        std::optional<ClauseTaskDepend> keywordDepend =
            symbolizeClauseTaskDepend(keyword);
        if (!keywordDepend)
          return failure();
        auto kindAttr =
            ClauseTaskDependAttr::get(parser.getContext(), *keywordDepend);
        if (llvm::isa<mlir::omp::IteratedType>(ty)) {
          iteratedVars.push_back(operand);
          iteratedTypes.push_back(ty);
          iterKindsVec.push_back(kindAttr);
        } else {
          dependVars.push_back(operand);
          dependTypes.push_back(ty);
          kindsVec.push_back(kindAttr);
        }
        return success();
      })))
    return failure();
  SmallVector<Attribute> kinds(kindsVec.begin(), kindsVec.end());
  dependKinds = ArrayAttr::get(parser.getContext(), kinds);
  SmallVector<Attribute> iterKinds(iterKindsVec.begin(), iterKindsVec.end());
  iteratedKinds = ArrayAttr::get(parser.getContext(), iterKinds);
  return success();
}
 
/// Print Depend clause
static void printDependVarList(OpAsmPrinter &p, Operation *op,
                               OperandRange dependVars, TypeRange dependTypes,
                               std::optional<ArrayAttr> dependKinds,
                               OperandRange iteratedVars,
                               TypeRange iteratedTypes,
                               std::optional<ArrayAttr> iteratedKinds) {
  bool first = true;
  auto printEntries = [&](OperandRange vars, TypeRange types,
                          std::optional<ArrayAttr> kinds) {
    for (unsigned i = 0, e = vars.size(); i < e; ++i) {
      if (!first)
        p << ", ";
      p << stringifyClauseTaskDepend(
               llvm::cast<mlir::omp::ClauseTaskDependAttr>((*kinds)[i])
                   .getValue())
        << " -> " << vars[i] << " : " << types[i];
      first = false;
    }
  };
  printEntries(dependVars, dependTypes, dependKinds);
  printEntries(iteratedVars, iteratedTypes, iteratedKinds);
}
 
/// Verifies Depend clause
static LogicalResult verifyDependVarList(Operation *op,
                                         std::optional<ArrayAttr> dependKinds,
                                         OperandRange dependVars,
                                         std::optional<ArrayAttr> iteratedKinds,
                                         OperandRange iteratedVars) {
  if (!dependVars.empty()) {
    if (!dependKinds || dependKinds->size() != dependVars.size())
      return op->emitOpError() << "expected as many depend values"
                                  " as depend variables";
  } else {
    if (dependKinds && !dependKinds->empty())
      return op->emitOpError() << "unexpected depend values";
  }
 
  if (!iteratedVars.empty()) {
    if (!iteratedKinds || iteratedKinds->size() != iteratedVars.size())
      return op->emitOpError() << "expected as many depend iterated values"
                                  " as depend iterated variables";
  } else {
    if (iteratedKinds && !iteratedKinds->empty())
      return op->emitOpError() << "unexpected depend iterated values";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Parser, printer and verifier for Synchronization Hint (2.17.12)
//===----------------------------------------------------------------------===//
 
/// Parses a Synchronization Hint clause. The value of hint is an integer
/// which is a combination of different hints from `omp_sync_hint_t`.
///
/// hint-clause = `hint` `(` hint-value `)`
static ParseResult parseSynchronizationHint(OpAsmParser &parser,
                                            IntegerAttr &hintAttr) {
  StringRef hintKeyword;
  int64_t hint = 0;
  if (succeeded(parser.parseOptionalKeyword("none"))) {
    hintAttr = IntegerAttr::get(parser.getBuilder().getI64Type(), 0);
    return success();
  }
  auto parseKeyword = [&]() -> ParseResult {
    if (failed(parser.parseKeyword(&hintKeyword)))
      return failure();
    if (hintKeyword == "uncontended")
      hint |= 1;
    else if (hintKeyword == "contended")
      hint |= 2;
    else if (hintKeyword == "nonspeculative")
      hint |= 4;
    else if (hintKeyword == "speculative")
      hint |= 8;
    else
      return parser.emitError(parser.getCurrentLocation())
             << hintKeyword << " is not a valid hint";
    return success();
  };
  if (parser.parseCommaSeparatedList(parseKeyword))
    return failure();
  hintAttr = IntegerAttr::get(parser.getBuilder().getI64Type(), hint);
  return success();
}
 
/// Prints a Synchronization Hint clause
static void printSynchronizationHint(OpAsmPrinter &p, Operation *op,
                                     IntegerAttr hintAttr) {
  int64_t hint = hintAttr.getInt();
 
  if (hint == 0) {
    p << "none";
    return;
  }
 
  // Helper function to get n-th bit from the right end of `value`
  auto bitn = [](int value, int n) -> bool { return value & (1 << n); };
 
  bool uncontended = bitn(hint, 0);
  bool contended = bitn(hint, 1);
  bool nonspeculative = bitn(hint, 2);
  bool speculative = bitn(hint, 3);
 
  SmallVector<StringRef> hints;
  if (uncontended)
    hints.push_back("uncontended");
  if (contended)
    hints.push_back("contended");
  if (nonspeculative)
    hints.push_back("nonspeculative");
  if (speculative)
    hints.push_back("speculative");
 
  llvm::interleaveComma(hints, p);
}
 
/// Verifies a synchronization hint clause
static LogicalResult verifySynchronizationHint(Operation *op, uint64_t hint) {
 
  // Helper function to get n-th bit from the right end of `value`
  auto bitn = [](int value, int n) -> bool { return value & (1 << n); };
 
  bool uncontended = bitn(hint, 0);
  bool contended = bitn(hint, 1);
  bool nonspeculative = bitn(hint, 2);
  bool speculative = bitn(hint, 3);
 
  if (uncontended && contended)
    return op->emitOpError() << "the hints omp_sync_hint_uncontended and "
                                "omp_sync_hint_contended cannot be combined";
  if (nonspeculative && speculative)
    return op->emitOpError() << "the hints omp_sync_hint_nonspeculative and "
                                "omp_sync_hint_speculative cannot be combined.";
  return success();
}
 
//===----------------------------------------------------------------------===//
// Parser, printer and verifier for Target
//===----------------------------------------------------------------------===//
 
// Helper function to get bitwise AND of `value` and 'flag' then return it as a
// boolean
static bool mapTypeToBool(ClauseMapFlags value, ClauseMapFlags flag) {
  return (value & flag) == flag;
}
 
/// Parses a map_entries map type from a string format back into its numeric
/// value.
///
/// map-clause = `map_clauses (  ( `(` `always, `? `implicit, `? `ompx_hold, `?
/// `close, `? `present, `? ( `to` | `from` | `delete` `)` )+ `)` )
static ParseResult parseMapClause(OpAsmParser &parser,
                                  ClauseMapFlagsAttr &mapType) {
  ClauseMapFlags mapTypeBits = ClauseMapFlags::none;
  // This simply verifies the correct keyword is read in, the
  // keyword itself is stored inside of the operation
  auto parseTypeAndMod = [&]() -> ParseResult {
    StringRef mapTypeMod;
    if (parser.parseKeyword(&mapTypeMod))
      return failure();
 
    if (mapTypeMod == "always")
      mapTypeBits |= ClauseMapFlags::always;
 
    if (mapTypeMod == "implicit")
      mapTypeBits |= ClauseMapFlags::implicit;
 
    if (mapTypeMod == "ompx_hold")
      mapTypeBits |= ClauseMapFlags::ompx_hold;
 
    if (mapTypeMod == "close")
      mapTypeBits |= ClauseMapFlags::close;
 
    if (mapTypeMod == "present")
      mapTypeBits |= ClauseMapFlags::present;
 
    if (mapTypeMod == "to")
      mapTypeBits |= ClauseMapFlags::to;
 
    if (mapTypeMod == "from")
      mapTypeBits |= ClauseMapFlags::from;
 
    if (mapTypeMod == "tofrom")
      mapTypeBits |= ClauseMapFlags::to | ClauseMapFlags::from;
 
    if (mapTypeMod == "delete")
      mapTypeBits |= ClauseMapFlags::del;
 
    if (mapTypeMod == "storage")
      mapTypeBits |= ClauseMapFlags::storage;
 
    if (mapTypeMod == "return_param")
      mapTypeBits |= ClauseMapFlags::return_param;
 
    if (mapTypeMod == "private")
      mapTypeBits |= ClauseMapFlags::priv;
 
    if (mapTypeMod == "literal")
      mapTypeBits |= ClauseMapFlags::literal;
 
    if (mapTypeMod == "attach")
      mapTypeBits |= ClauseMapFlags::attach;
 
    if (mapTypeMod == "attach_always")
      mapTypeBits |= ClauseMapFlags::attach_always;
 
    if (mapTypeMod == "attach_never")
      mapTypeBits |= ClauseMapFlags::attach_never;
 
    if (mapTypeMod == "attach_auto")
      mapTypeBits |= ClauseMapFlags::attach_auto;
 
    if (mapTypeMod == "ref_ptr")
      mapTypeBits |= ClauseMapFlags::ref_ptr;
 
    if (mapTypeMod == "ref_ptee")
      mapTypeBits |= ClauseMapFlags::ref_ptee;
 
    if (mapTypeMod == "is_device_ptr")
      mapTypeBits |= ClauseMapFlags::is_device_ptr;
 
    if (mapTypeMod == "target_param")
      mapTypeBits |= ClauseMapFlags::target_param;
 
    return success();
  };
 
  if (parser.parseCommaSeparatedList(parseTypeAndMod))
    return failure();
 
  mapType =
      parser.getBuilder().getAttr<mlir::omp::ClauseMapFlagsAttr>(mapTypeBits);
 
  return success();
}
 
/// Prints a map_entries map type from its numeric value out into its string
/// format.
static void printMapClause(OpAsmPrinter &p, Operation *op,
                           ClauseMapFlagsAttr mapType) {
  llvm::SmallVector<std::string, 4> mapTypeStrs;
  ClauseMapFlags mapFlags = mapType.getValue();
 
  // handling of always, close, present placed at the beginning of the string
  // to aid readability
  if (mapTypeToBool(mapFlags, ClauseMapFlags::always))
    mapTypeStrs.push_back("always");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::implicit))
    mapTypeStrs.push_back("implicit");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::ompx_hold))
    mapTypeStrs.push_back("ompx_hold");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::close))
    mapTypeStrs.push_back("close");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::present))
    mapTypeStrs.push_back("present");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::target_param))
    mapTypeStrs.push_back("target_param");
 
  // special handling of to/from/tofrom/delete and release/alloc, release +
  // alloc are the abscense of one of the other flags, whereas tofrom requires
  // both the to and from flag to be set.
  bool to = mapTypeToBool(mapFlags, ClauseMapFlags::to);
  bool from = mapTypeToBool(mapFlags, ClauseMapFlags::from);
 
  if (to && from)
    mapTypeStrs.push_back("tofrom");
  else if (from)
    mapTypeStrs.push_back("from");
  else if (to)
    mapTypeStrs.push_back("to");
 
  if (mapTypeToBool(mapFlags, ClauseMapFlags::del))
    mapTypeStrs.push_back("delete");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::return_param))
    mapTypeStrs.push_back("return_param");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::storage))
    mapTypeStrs.push_back("storage");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::priv))
    mapTypeStrs.push_back("private");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::literal))
    mapTypeStrs.push_back("literal");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach))
    mapTypeStrs.push_back("attach");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_always))
    mapTypeStrs.push_back("attach_always");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_never))
    mapTypeStrs.push_back("attach_never");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::attach_auto))
    mapTypeStrs.push_back("attach_auto");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptr))
    mapTypeStrs.push_back("ref_ptr");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::ref_ptee))
    mapTypeStrs.push_back("ref_ptee");
  if (mapTypeToBool(mapFlags, ClauseMapFlags::is_device_ptr))
    mapTypeStrs.push_back("is_device_ptr");
  if (mapFlags == ClauseMapFlags::none)
    mapTypeStrs.push_back("none");
 
  for (unsigned int i = 0; i < mapTypeStrs.size(); ++i) {
    p << mapTypeStrs[i];
    if (i + 1 < mapTypeStrs.size()) {
      p << ", ";
    }
  }
}
 
static ParseResult parseMembersIndex(OpAsmParser &parser,
                                     ArrayAttr &membersIdx) {
  SmallVector<Attribute> values, memberIdxs;
 
  auto parseIndices = [&]() -> ParseResult {
    int64_t value;
    if (parser.parseInteger(value))
      return failure();
    values.push_back(IntegerAttr::get(parser.getBuilder().getIntegerType(64),
                                      APInt(64, value, /*isSigned=*/false)));
    return success();
  };
 
  do {
    if (failed(parser.parseLSquare()))
      return failure();
 
    if (parser.parseCommaSeparatedList(parseIndices))
      return failure();
 
    if (failed(parser.parseRSquare()))
      return failure();
 
    memberIdxs.push_back(ArrayAttr::get(parser.getContext(), values));
    values.clear();
  } while (succeeded(parser.parseOptionalComma()));
 
  if (!memberIdxs.empty())
    membersIdx = ArrayAttr::get(parser.getContext(), memberIdxs);
 
  return success();
}
 
static void printMembersIndex(OpAsmPrinter &p, MapInfoOp op,
                              ArrayAttr membersIdx) {
  if (!membersIdx)
    return;
 
  llvm::interleaveComma(membersIdx, p, [&p](Attribute v) {
    p << "[";
    auto memberIdx = cast<ArrayAttr>(v);
    llvm::interleaveComma(memberIdx.getValue(), p, [&p](Attribute v2) {
      p << cast<IntegerAttr>(v2).getInt();
    });
    p << "]";
  });
}
 
static void printCaptureType(OpAsmPrinter &p, Operation *op,
                             VariableCaptureKindAttr mapCaptureType) {
  std::string typeCapStr;
  llvm::raw_string_ostream typeCap(typeCapStr);
  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByRef)
    typeCap << "ByRef";
  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::ByCopy)
    typeCap << "ByCopy";
  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::VLAType)
    typeCap << "VLAType";
  if (mapCaptureType.getValue() == mlir::omp::VariableCaptureKind::This)
    typeCap << "This";
  p << typeCapStr;
}
 
static ParseResult parseCaptureType(OpAsmParser &parser,
                                    VariableCaptureKindAttr &mapCaptureType) {
  StringRef mapCaptureKey;
  if (parser.parseKeyword(&mapCaptureKey))
    return failure();
 
  if (mapCaptureKey == "This")
    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
        parser.getContext(), mlir::omp::VariableCaptureKind::This);
  if (mapCaptureKey == "ByRef")
    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
        parser.getContext(), mlir::omp::VariableCaptureKind::ByRef);
  if (mapCaptureKey == "ByCopy")
    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
        parser.getContext(), mlir::omp::VariableCaptureKind::ByCopy);
  if (mapCaptureKey == "VLAType")
    mapCaptureType = mlir::omp::VariableCaptureKindAttr::get(
        parser.getContext(), mlir::omp::VariableCaptureKind::VLAType);
 
  return success();
}
 
static LogicalResult verifyMapInfoForMapClause(
    Operation *op, mlir::omp::MapInfoOp mapInfoOp,
    llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> &updateToVars,
    llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>>
        &updateFromVars) {
  mlir::omp::ClauseMapFlags mapTypeBits = mapInfoOp.getMapType();
 
  bool to = mapTypeToBool(mapTypeBits, ClauseMapFlags::to);
  bool from = mapTypeToBool(mapTypeBits, ClauseMapFlags::from);
  bool del = mapTypeToBool(mapTypeBits, ClauseMapFlags::del);
 
  bool always = mapTypeToBool(mapTypeBits, ClauseMapFlags::always);
  bool close = mapTypeToBool(mapTypeBits, ClauseMapFlags::close);
  bool implicit = mapTypeToBool(mapTypeBits, ClauseMapFlags::implicit);
  bool attach = mapTypeToBool(mapTypeBits, ClauseMapFlags::attach);
 
  if ((isa<TargetDataOp>(op) || isa<TargetOp>(op)) && del)
    return emitError(op->getLoc(),
                     "to, from, tofrom and alloc map types are permitted");
 
  if (isa<TargetEnterDataOp>(op) && (from || del))
    return emitError(op->getLoc(), "to and alloc map types are permitted");
 
  if (isa<TargetExitDataOp>(op) && to)
    return emitError(op->getLoc(),
                     "from, release and delete map types are permitted");
 
  if (isa<TargetUpdateOp>(op)) {
    if (del) {
      return emitError(op->getLoc(),
                       "at least one of to or from map types must be "
                       "specified, other map types are not permitted");
    }
 
    if (!to && !from && !attach) {
      return emitError(op->getLoc(),
                       "at least one of to or from or attach map types must be "
                       "specified, other map types are not permitted");
    }
 
    auto updateVar = mapInfoOp.getVarPtr();
 
    if ((to && from) || (to && updateFromVars.contains(updateVar)) ||
        (from && updateToVars.contains(updateVar))) {
      return emitError(
          op->getLoc(),
          "either to or from map types can be specified, not both");
    }
 
    if (always || close || implicit) {
      return emitError(
          op->getLoc(),
          "present, mapper and iterator map type modifiers are permitted");
    }
 
    // It's possible we have an attach map, in which case if there is no to
    // or from tied to it, we skip insertion.
    if (to || from) {
      to ? updateToVars.insert(updateVar) : updateFromVars.insert(updateVar);
    }
  }
 
  if ((mapInfoOp.getVarPtrPtr() && !mapInfoOp.getVarPtrPtrType()) ||
      (!mapInfoOp.getVarPtrPtr() && mapInfoOp.getVarPtrPtrType())) {
    return emitError(op->getLoc(),
                     "if varPtrPtr or varPtrPtrType is specified, then both "
                     "must be present");
  }
 
  return success();
}
 
static LogicalResult verifyMapClause(Operation *op, OperandRange mapVars,
                                     OperandRange mapIterated) {
  llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateToVars;
  llvm::DenseSet<mlir::TypedValue<mlir::omp::PointerLikeType>> updateFromVars;
 
  for (auto mapOp : mapVars) {
    if (!mapOp.getDefiningOp())
      return emitError(op->getLoc(), "missing map operation");
 
    if (auto mapInfoOp = mapOp.getDefiningOp<mlir::omp::MapInfoOp>()) {
      if (failed(verifyMapInfoForMapClause(op, mapInfoOp, updateToVars,
                                           updateFromVars)))
        return failure();
    } else if (!isa<DeclareMapperInfoOp>(op)) {
      return emitError(op->getLoc(),
                       "map argument is not a map entry operation");
    }
  }
 
  // Verify iterated map entries.
  for (auto iterVal : mapIterated) {
    auto iterOp = iterVal.getDefiningOp<mlir::omp::IteratorOp>();
    if (!iterOp)
      return op->emitOpError() << "'map_iterated' arguments must be defined by "
                                  "'omp.iterator' ops";
 
    // Check that the iterator body yields a value defined by omp.map.info.
    mlir::omp::YieldOp yieldOp = iterOp.getYieldOp();
    auto yieldedMapInfo =
        yieldOp && yieldOp.getNumOperands() == 1
            ? yieldOp.getResults()[0].getDefiningOp<mlir::omp::MapInfoOp>()
            : nullptr;
    if (!yieldedMapInfo)
      return op->emitOpError() << "'map_iterated' iterator body must yield "
                                  "a value defined by 'omp.map.info'";
 
    if (failed(verifyMapInfoForMapClause(op, yieldedMapInfo, updateToVars,
                                         updateFromVars)))
      return failure();
  }
 
  return success();
}
 
template <typename OpType>
static LogicalResult verifyPrivateVarList(OpType &op);
 
static LogicalResult verifyPrivateVarsMapping(TargetOp targetOp) {
  std::optional<DenseI64ArrayAttr> privateMapIndices =
      targetOp.getPrivateMapsAttr();
 
  // None of the private operands are mapped.
  if (!privateMapIndices.has_value() || !privateMapIndices.value())
    return success();
 
  OperandRange privateVars = targetOp.getPrivateVars();
 
  if (privateMapIndices.value().size() !=
      static_cast<int64_t>(privateVars.size()))
    return emitError(targetOp.getLoc(), "sizes of `private` operand range and "
                                        "`private_maps` attribute mismatch");
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// MapInfoOp
//===----------------------------------------------------------------------===//
 
static LogicalResult verifyMapInfoDefinedArgs(Operation *op,
                                              StringRef clauseName,
                                              OperandRange vars) {
  for (Value var : vars)
    if (!llvm::isa_and_present<MapInfoOp>(var.getDefiningOp()))
      return op->emitOpError()
             << "'" << clauseName
             << "' arguments must be defined by 'omp.map.info' ops";
  return success();
}
 
LogicalResult MapInfoOp::verify() {
  if (getMapperId() &&
      !SymbolTable::lookupNearestSymbolFrom<omp::DeclareMapperOp>(
          *this, getMapperIdAttr())) {
    return emitError("invalid mapper id");
  }
 
  if (failed(verifyMapInfoDefinedArgs(*this, "members", getMembers())))
    return failure();
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// TargetDataOp
//===----------------------------------------------------------------------===//
 
void TargetDataOp::build(OpBuilder &builder, OperationState &state,
                         const TargetDataOperands &clauses) {
  TargetDataOp::build(builder, state, clauses.device, clauses.ifExpr,
                      clauses.mapVars, clauses.mapIterated,
                      clauses.useDeviceAddrVars, clauses.useDevicePtrVars);
}
 
LogicalResult TargetDataOp::verify() {
  if (getMapVars().empty() && getMapIterated().empty() &&
      getUseDevicePtrVars().empty() && getUseDeviceAddrVars().empty()) {
    return ::emitError(this->getLoc(),
                       "At least one of map, use_device_ptr_vars, or "
                       "use_device_addr_vars operand must be present");
  }
 
  if (failed(verifyMapInfoDefinedArgs(*this, "use_device_ptr",
                                      getUseDevicePtrVars())))
    return failure();
 
  if (failed(verifyMapInfoDefinedArgs(*this, "use_device_addr",
                                      getUseDeviceAddrVars())))
    return failure();
 
  return verifyMapClause(*this, getMapVars(), getMapIterated());
}
 
//===----------------------------------------------------------------------===//
// TargetEnterDataOp
//===----------------------------------------------------------------------===//
 
void TargetEnterDataOp::build(
    OpBuilder &builder, OperationState &state,
    const TargetEnterExitUpdateDataOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TargetEnterDataOp::build(
      builder, state, makeArrayAttr(ctx, clauses.dependKinds),
      clauses.dependVars, makeArrayAttr(ctx, clauses.dependIteratedKinds),
      clauses.dependIterated, clauses.device, clauses.ifExpr, clauses.mapVars,
      clauses.mapIterated, clauses.nowait);
}
 
LogicalResult TargetEnterDataOp::verify() {
  LogicalResult verifyDependVars =
      verifyDependVarList(*this, getDependKinds(), getDependVars(),
                          getDependIteratedKinds(), getDependIterated());
  return failed(verifyDependVars)
             ? verifyDependVars
             : verifyMapClause(*this, getMapVars(), getMapIterated());
}
 
//===----------------------------------------------------------------------===//
// TargetExitDataOp
//===----------------------------------------------------------------------===//
 
void TargetExitDataOp::build(OpBuilder &builder, OperationState &state,
                             const TargetEnterExitUpdateDataOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TargetExitDataOp::build(
      builder, state, makeArrayAttr(ctx, clauses.dependKinds),
      clauses.dependVars, makeArrayAttr(ctx, clauses.dependIteratedKinds),
      clauses.dependIterated, clauses.device, clauses.ifExpr, clauses.mapVars,
      clauses.mapIterated, clauses.nowait);
}
 
LogicalResult TargetExitDataOp::verify() {
  LogicalResult verifyDependVars =
      verifyDependVarList(*this, getDependKinds(), getDependVars(),
                          getDependIteratedKinds(), getDependIterated());
  return failed(verifyDependVars)
             ? verifyDependVars
             : verifyMapClause(*this, getMapVars(), getMapIterated());
}
 
//===----------------------------------------------------------------------===//
// TargetUpdateOp
//===----------------------------------------------------------------------===//
 
void TargetUpdateOp::build(OpBuilder &builder, OperationState &state,
                           const TargetEnterExitUpdateDataOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TargetUpdateOp::build(builder, state, makeArrayAttr(ctx, clauses.dependKinds),
                        clauses.dependVars,
                        makeArrayAttr(ctx, clauses.dependIteratedKinds),
                        clauses.dependIterated, clauses.device, clauses.ifExpr,
                        clauses.mapVars, clauses.mapIterated, clauses.nowait);
}
 
LogicalResult TargetUpdateOp::verify() {
  LogicalResult verifyDependVars =
      verifyDependVarList(*this, getDependKinds(), getDependVars(),
                          getDependIteratedKinds(), getDependIterated());
  return failed(verifyDependVars)
             ? verifyDependVars
             : verifyMapClause(*this, getMapVars(), getMapIterated());
}
 
//===----------------------------------------------------------------------===//
// TargetOp
//===----------------------------------------------------------------------===//
 
void TargetOp::build(OpBuilder &builder, OperationState &state,
                     const TargetExtOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TargetOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
      makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,
      makeArrayAttr(ctx, clauses.dependIteratedKinds), clauses.dependIterated,
      clauses.device, clauses.dynGroupprivateAccessGroup,
      clauses.dynGroupprivateFallback, clauses.dynGroupprivateSize,
      clauses.hasDeviceAddrVars, clauses.hostEvalVars, clauses.ifExpr,
      clauses.inReductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
      makeArrayAttr(ctx, clauses.inReductionSyms), clauses.isDevicePtrVars,
      clauses.mapVars, clauses.mapIterated, clauses.nowait, clauses.privateVars,
      makeArrayAttr(ctx, clauses.privateSyms), clauses.privateNeedsBarrier,
      clauses.threadLimitVars, /*private_maps=*/nullptr, clauses.kernelType);
}
 
bool TargetOp::hasHostEvalTripCount() {
  TargetExecMode mode = getKernelType();
  if (mode == TargetExecMode::spmd || mode == TargetExecMode::spmd_no_loop)
    return true;
 
  if (mode == TargetExecMode::bare)
    return false;
 
  // If it represents a `target teams distribute` construct, also evaluate the
  // `distribute` trip count on the host.
  Operation *capturedOp =
      cast<ComposableOpInterface>(getOperation()).findCapturedOp();
  if (auto loopNestOp = dyn_cast_if_present<LoopNestOp>(capturedOp)) {
    SmallVector<LoopWrapperInterface> loopWrappers;
    loopNestOp.gatherWrappers(loopWrappers);
 
    LoopWrapperInterface *innermostWrapper = loopWrappers.begin();
    if (isa<SimdOp>(innermostWrapper))
      innermostWrapper = std::next(innermostWrapper);
 
    auto numWrappers = std::distance(innermostWrapper, loopWrappers.end());
    if (numWrappers != 1)
      return false;
 
    if (!isa<DistributeOp>(innermostWrapper))
      return false;
 
    Operation *parentOp = innermostWrapper->getOperation()->getParentOp();
    if (isa_and_present<TeamsOp>(parentOp) &&
        parentOp->getParentOp() == getOperation())
      return true;
  }
 
  return false;
}
 
/// An `omp.target` `in_reduction` operand is captured by a `map_entries` entry
/// when the entry's `MapInfoOp` var_ptr is the same SSA value, or another
/// result of the same defining op. At this stage, exact identity can only be
/// required for block arguments, which have no defining op. Flang emits
/// `hlfir.declare` #0 for the `in_reduction` operand and #1 for the map
/// `var_ptr`; these collapse to the same value after lowering, but that cannot
/// be enforced here.
static bool targetInReductionCapturedBy(Value inReductionVar, Value mapVarPtr) {
  if (mapVarPtr == inReductionVar)
    return true;
  Operation *def = inReductionVar.getDefiningOp();
  return def && mapVarPtr.getDefiningOp() == def;
}
 
LogicalResult TargetOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();
 
  if (getKernelType() == TargetExecMode::bare && !isCombined())
    return emitOpError() << "bare kernel requires 'omp.combined'";
 
  if (failed(verifyDependVarList(*this, getDependKinds(), getDependVars(),
                                 getDependIteratedKinds(),
                                 getDependIterated())))
    return failure();
 
  if (failed(verifyMapInfoDefinedArgs(*this, "has_device_addr",
                                      getHasDeviceAddrVars())))
    return failure();
 
  if (failed(verifyMapClause(*this, getMapVars(), getMapIterated())))
    return failure();
 
  if (failed(verifyDynGroupprivateClause(
          *this, getDynGroupprivateAccessGroupAttr(),
          getDynGroupprivateFallbackAttr(), getDynGroupprivateSize())))
    return failure();
 
  if (failed(verifyPrivateVarList(*this)))
    return failure();
 
  if (failed(verifyReductionVarList(*this, getInReductionSyms(),
                                    getInReductionVars(),
                                    getInReductionByref())))
    return failure();
 
  // An `in_reduction` operand on `omp.target` has no dedicated entry block
  // argument; inside the region it is accessed through the block argument of a
  // matching `map_entries` entry, and the host rewrites that map argument to
  // the reduction-private storage. Require every `in_reduction` operand to be
  // captured by at least one `map_entries` entry.
  for (Value inReductionVar : getInReductionVars()) {
    bool captured = false;
    for (Value mapVar : getMapVars()) {
      auto mapInfo = mapVar.getDefiningOp<MapInfoOp>();
      if (targetInReductionCapturedBy(inReductionVar, mapInfo.getVarPtr())) {
        captured = true;
        break;
      }
    }
    if (!captured)
      return emitOpError() << "in_reduction variable must be captured by a "
                              "matching map_entries entry";
  }
 
  return verifyPrivateVarsMapping(*this);
}
 
LogicalResult TargetOp::verifyRegions() {
  auto teamsOps = getOps<TeamsOp>();
  auto numNestedTeams = std::distance(teamsOps.begin(), teamsOps.end());
  if (numNestedTeams > 1)
    return emitError("target containing multiple 'omp.teams' nested ops");
 
  if (numNestedTeams == 0) {
    switch (getKernelType()) {
    case TargetExecMode::bare:
      return emitOpError()
             << "bare kernel must contain a nested 'omp.teams' operation";
    case TargetExecMode::spmd_no_loop:
      return emitOpError() << "spmd_no_loop kernel must contain a nested "
                              "'omp.teams' operation";
    default:
      break;
    }
  }
 
  Operation *capturedOp =
      cast<ComposableOpInterface>(getOperation()).findCapturedOp();
  if ((getKernelType() == TargetExecMode::spmd ||
       getKernelType() == TargetExecMode::spmd_no_loop) &&
      !isa_and_present<LoopNestOp>(capturedOp))
    return emitOpError()
           << "SPMD kernel must capture an 'omp.loop_nest' operation";
 
  bool isTargetDevice = false;
  if (auto offloadMod = (*this)->getParentOfType<OffloadModuleInterface>())
    if (offloadMod.getIsTargetDevice())
      isTargetDevice = true;
 
  // Check that host_eval values are only used in legal ways.
  llvm::ArrayRef<BlockArgument> hostEvalBlockArgs =
      cast<BlockArgOpenMPOpInterface>(getOperation()).getHostEvalBlockArgs();
 
  bool hostEvalTripCount = hasHostEvalTripCount();
  for (Value hostEvalArg : hostEvalBlockArgs) {
    for (Operation *user : hostEvalArg.getUsers()) {
      if (auto teamsOp = dyn_cast<TeamsOp>(user)) {
        // Check if used in num_teams_lower or any of num_teams_upper_vars
        if (hostEvalArg == teamsOp.getNumTeamsLower() ||
            llvm::is_contained(teamsOp.getNumTeamsUpperVars(), hostEvalArg) ||
            llvm::is_contained(teamsOp.getThreadLimitVars(), hostEvalArg))
          continue;
 
        return emitOpError() << "host_eval argument only legal as 'num_teams' "
                                "and 'thread_limit' in 'omp.teams'";
      }
      if (auto parallelOp = dyn_cast<ParallelOp>(user)) {
        if (llvm::is_contained(parallelOp.getNumThreadsVars(), hostEvalArg))
          continue;
 
        return emitOpError()
               << "host_eval argument only legal as 'num_threads' in "
                  "'omp.parallel'";
      }
      if (auto loopNestOp = dyn_cast<LoopNestOp>(user)) {
        if (hostEvalTripCount &&
            (llvm::is_contained(loopNestOp.getLoopLowerBounds(), hostEvalArg) ||
             llvm::is_contained(loopNestOp.getLoopUpperBounds(), hostEvalArg) ||
             llvm::is_contained(loopNestOp.getLoopSteps(), hostEvalArg)))
          continue;
 
        return emitOpError() << "host_eval argument only legal as loop bounds "
                                "and steps in 'omp.loop_nest' when trip count "
                                "must be evaluated in the host";
      }
 
      return emitOpError() << "host_eval argument illegal use in '"
                           << user->getName() << "' operation";
    }
  }
 
  if (hostEvalTripCount && !isTargetDevice) {
    auto loopOp = cast<LoopNestOp>(capturedOp);
    for (auto arg : llvm::concat<Value>(loopOp.getLoopLowerBounds(),
                                        loopOp.getLoopUpperBounds(),
                                        loopOp.getLoopSteps())) {
      if (!llvm::is_contained(hostEvalBlockArgs, arg))
        return emitOpError() << "nested 'omp.loop_nest' bounds expected to "
                                "be host-evaluated";
    }
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// ParallelOp
//===----------------------------------------------------------------------===//
 
void ParallelOp::build(OpBuilder &builder, OperationState &state,
                       ArrayRef<NamedAttribute> attributes) {
  ParallelOp::build(builder, state, /*allocate_vars=*/ValueRange(),
                    /*allocator_vars=*/ValueRange(),
                    /*allocate_alignments=*/nullptr,
                    /*allocate_private_indices=*/nullptr, /*if_expr=*/nullptr,
                    /*num_threads_vars=*/ValueRange(),
                    /*private_vars=*/ValueRange(),
                    /*private_syms=*/nullptr, /*private_needs_barrier=*/false,
                    /*proc_bind_kind=*/nullptr,
                    /*reduction_mod =*/nullptr, /*reduction_vars=*/ValueRange(),
                    /*reduction_byref=*/nullptr, /*reduction_syms=*/nullptr);
  state.addAttributes(attributes);
}
 
void ParallelOp::build(OpBuilder &builder, OperationState &state,
                       const ParallelOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  ParallelOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
                    makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                    makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                    clauses.ifExpr, clauses.numThreadsVars, clauses.privateVars,
                    makeArrayAttr(ctx, clauses.privateSyms),
                    clauses.privateNeedsBarrier, clauses.procBindKind,
                    clauses.reductionMod, clauses.reductionVars,
                    makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                    makeArrayAttr(ctx, clauses.reductionSyms));
}
 
template <typename OpType>
static LogicalResult verifyPrivateVarList(OpType &op) {
  auto privateVars = op.getPrivateVars();
  auto privateSyms = op.getPrivateSymsAttr();
 
  if (privateVars.empty() && (privateSyms == nullptr || privateSyms.empty()))
    return success();
 
  auto numPrivateVars = privateVars.size();
  auto numPrivateSyms = (privateSyms == nullptr) ? 0 : privateSyms.size();
 
  if (numPrivateVars != numPrivateSyms)
    return op.emitError() << "inconsistent number of private variables and "
                             "privatizer op symbols, private vars: "
                          << numPrivateVars
                          << " vs. privatizer op symbols: " << numPrivateSyms;
 
  for (auto privateVarInfo : llvm::zip_equal(privateVars, privateSyms)) {
    Type varType = std::get<0>(privateVarInfo).getType();
    SymbolRefAttr privateSym = cast<SymbolRefAttr>(std::get<1>(privateVarInfo));
    PrivateClauseOp privatizerOp =
        SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, privateSym);
 
    if (privatizerOp == nullptr)
      return op.emitError() << "failed to lookup privatizer op with symbol: '"
                            << privateSym << "'";
 
    Type privatizerType = privatizerOp.getArgType();
 
    if (privatizerType && (varType != privatizerType))
      return op.emitError()
             << "type mismatch between a "
             << stringifyDataSharingClauseType(
                    privatizerOp.getDataSharingType())
             << " variable and its privatizer op, var type: " << varType
             << " vs. privatizer op type: " << privatizerType;
 
    if (privatizerOp.getDataSharingType() == DataSharingClauseType::Linear)
      return op.emitError() << "`linear` privatizer '" << privateSym
                            << "' can only be used in a `linear` clause";
  }
 
  return success();
}
 
/// Verifies that each linear variable has a `linear` privatizer.
template <typename OpType>
static LogicalResult verifyLinearSyms(OpType op) {
  OperandRange linearVars = op.getLinearVars();
  std::optional<ArrayAttr> linearSyms = op.getLinearSyms();
  if (linearVars.empty() && (!linearSyms || linearSyms->empty()))
    return success();
 
  if (!linearSyms || linearSyms->size() != linearVars.size())
    return op.emitOpError()
           << "expected as many linear privatizers as linear variables";
 
  for (auto [var, sym] : llvm::zip_equal(linearVars, *linearSyms)) {
    auto symRef = cast<SymbolRefAttr>(sym);
    auto privatizer =
        SymbolTable::lookupNearestSymbolFrom<PrivateClauseOp>(op, symRef);
    if (!privatizer)
      return op.emitOpError() << "failed to lookup privatizer op with symbol: '"
                              << symRef << "'";
    if (privatizer.getDataSharingType() != DataSharingClauseType::Linear)
      return op.emitOpError() << "privatizer '" << symRef
                              << "' of a linear variable must be `linear`";
    Type argType = privatizer.getArgType();
    if (argType && argType != var.getType())
      return op.emitOpError()
             << "type mismatch between a linear variable and its privatizer "
                "op, var type: "
             << var.getType() << " vs. privatizer op type: " << argType;
  }
  return success();
}
 
LogicalResult ParallelOp::verify() {
  if (failed(verifyPrivateVarList(*this)))
    return failure();
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr(),
          /*requirePrivateIndices=*/true)))
    return failure();
 
  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}
 
LogicalResult ParallelOp::verifyRegions() {
  auto distChildOps = getOps<DistributeOp>();
  int numDistChildOps = std::distance(distChildOps.begin(), distChildOps.end());
  if (numDistChildOps > 1)
    return emitError()
           << "multiple 'omp.distribute' nested inside of 'omp.parallel'";
 
  if (numDistChildOps == 1) {
    if (!isComposite())
      return emitError()
             << "'omp.composite' attribute missing from composite operation";
 
    auto *ompDialect = getContext()->getLoadedDialect<OpenMPDialect>();
    Operation &distributeOp = **distChildOps.begin();
    for (Operation &childOp : getOps()) {
      if (&childOp == &distributeOp || ompDialect != childOp.getDialect())
        continue;
 
      if (!childOp.hasTrait<OpTrait::IsTerminator>())
        return emitError() << "unexpected OpenMP operation inside of composite "
                              "'omp.parallel': "
                           << childOp.getName();
    }
  } else if (isComposite()) {
    return emitError()
           << "'omp.composite' attribute present in non-composite operation";
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// TeamsOp
//===----------------------------------------------------------------------===//
 
static bool opInGlobalImplicitParallelRegion(Operation *op) {
  while ((op = op->getParentOp()))
    if (isa<OpenMPDialect>(op->getDialect()))
      return false;
  return true;
}
 
void TeamsOp::build(OpBuilder &builder, OperationState &state,
                    const TeamsOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
  TeamsOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
      clauses.dynGroupprivateAccessGroup, clauses.dynGroupprivateFallback,
      clauses.dynGroupprivateSize, clauses.ifExpr, clauses.numTeamsLower,
      clauses.numTeamsUpperVars, /*private_vars=*/{}, /*private_syms=*/nullptr,
      /*private_needs_barrier=*/false, clauses.reductionMod,
      clauses.reductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
      makeArrayAttr(ctx, clauses.reductionSyms), clauses.threadLimitVars);
}
 
// Verify num_teams clause
static LogicalResult verifyNumTeamsClause(Operation *op, Value numTeamsLower,
                                          OperandRange numTeamsUpperVars) {
  // If lower is specified, upper must have exactly one value
  if (numTeamsLower) {
    if (numTeamsUpperVars.size() != 1)
      return op->emitError(
          "expected exactly one num_teams upper bound when lower bound is "
          "specified");
    if (numTeamsLower.getType() != numTeamsUpperVars[0].getType())
      return op->emitError(
          "expected num_teams upper bound and lower bound to be "
          "the same type");
  }
 
  return success();
}
 
LogicalResult TeamsOp::verify() {
  // Check parent region
  // TODO If nested inside of a target region, also check that it does not
  // contain any statements, declarations or directives other than this
  // omp.teams construct. The issue is how to support the initialization of
  // this operation's own arguments (allow SSA values across omp.target?).
  Operation *op = getOperation();
  auto parentTarget = llvm::dyn_cast_if_present<TargetOp>(op->getParentOp());
  if (!parentTarget && !opInGlobalImplicitParallelRegion(op))
    return emitError("expected to be nested inside of omp.target or not nested "
                     "in any OpenMP dialect operations");
 
  // Check for num_teams clause restrictions
  if (failed(verifyNumTeamsClause(op, this->getNumTeamsLower(),
                                  this->getNumTeamsUpperVars())))
    return failure();
 
  if (parentTarget &&
      parentTarget.getKernelType() == TargetExecMode::spmd_no_loop &&
      (getNumTeamsLower() || !getNumTeamsUpperVars().empty()))
    return emitOpError() << "'num_teams' not allowed in SPMD-no-loop kernels";
 
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();
 
  if (failed(verifyDynGroupprivateClause(
          op, getDynGroupprivateAccessGroupAttr(),
          getDynGroupprivateFallbackAttr(), getDynGroupprivateSize())))
    return failure();
 
  if (failed(verifyPrivateVarList(*this)))
    return failure();
 
  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}
 
//===----------------------------------------------------------------------===//
// SectionOp
//===----------------------------------------------------------------------===//
 
OperandRange SectionOp::getPrivateVars() {
  return getParentOp().getPrivateVars();
}
 
OperandRange SectionOp::getReductionVars() {
  return getParentOp().getReductionVars();
}
 
//===----------------------------------------------------------------------===//
// SectionsOp
//===----------------------------------------------------------------------===//
 
void SectionsOp::build(OpBuilder &builder, OperationState &state,
                       const SectionsOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
  SectionsOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
                    makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                    makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                    clauses.nowait, /*private_vars=*/{},
                    /*private_syms=*/nullptr, /*private_needs_barrier=*/nullptr,
                    clauses.reductionMod, clauses.reductionVars,
                    makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                    makeArrayAttr(ctx, clauses.reductionSyms));
}
 
LogicalResult SectionsOp::verify() {
  if (isCombined())
    return emitOpError() << "cannot be a non-innermost combined construct leaf";
 
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();
 
  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}
 
LogicalResult SectionsOp::verifyRegions() {
  for (auto &inst : *getRegion().begin()) {
    if (!(isa<SectionOp>(inst) || isa<TerminatorOp>(inst))) {
      return emitOpError()
             << "expected omp.section op or terminator op inside region";
    }
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// ScopeOp
//===----------------------------------------------------------------------===//
 
void ScopeOp::build(OpBuilder &builder, OperationState &state,
                    const ScopeOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  ScopeOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
                 makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                 makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                 clauses.nowait, clauses.privateVars,
                 makeArrayAttr(ctx, clauses.privateSyms),
                 clauses.privateNeedsBarrier, clauses.reductionMod,
                 clauses.reductionVars,
                 makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                 makeArrayAttr(ctx, clauses.reductionSyms));
}
 
LogicalResult ScopeOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr(),
          /*requirePrivateIndices=*/true)))
    return failure();
 
  if (failed(verifyPrivateVarList(*this)))
    return failure();
 
  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}
 
//===----------------------------------------------------------------------===//
// SingleOp
//===----------------------------------------------------------------------===//
 
void SingleOp::build(OpBuilder &builder, OperationState &state,
                     const SingleOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  // TODO Store clauses in op: privateVars, privateSyms, privateNeedsBarrier
  SingleOp::build(builder, state, clauses.allocateVars, clauses.allocatorVars,
                  makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                  makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                  clauses.copyprivateVars,
                  makeArrayAttr(ctx, clauses.copyprivateSyms), clauses.nowait,
                  /*private_vars=*/{}, /*private_syms=*/nullptr,
                  /*private_needs_barrier=*/nullptr);
}
 
LogicalResult SingleOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();
 
  return verifyCopyprivateVarList(*this, getCopyprivateVars(),
                                  getCopyprivateSyms());
}
 
//===----------------------------------------------------------------------===//
// WorkshareOp
//===----------------------------------------------------------------------===//
 
void WorkshareOp::build(OpBuilder &builder, OperationState &state,
                        const WorkshareOperands &clauses) {
  WorkshareOp::build(builder, state, clauses.nowait);
}
 
LogicalResult WorkshareOp::verify() {
  if (isCombined())
    return emitOpError() << "cannot be a non-innermost combined construct leaf";
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// WorkshareLoopWrapperOp
//===----------------------------------------------------------------------===//
 
LogicalResult WorkshareLoopWrapperOp::verifyRegions() {
  if (isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||
      getNestedWrapper())
    return emitOpError() << "expected to be a standalone loop wrapper";
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// LoopWrapperInterface
//===----------------------------------------------------------------------===//
 
LogicalResult LoopWrapperInterface::verifyImpl() {
  Operation *op = this->getOperation();
  if (!op->hasTrait<OpTrait::NoTerminator>() ||
      !op->hasTrait<OpTrait::SingleBlock>())
    return emitOpError() << "loop wrapper must also have the `NoTerminator` "
                            "and `SingleBlock` traits";
 
  if (op->getNumRegions() != 1)
    return emitOpError() << "loop wrapper does not contain exactly one region";
 
  Region &region = op->getRegion(0);
  if (range_size(region.getOps()) != 1)
    return emitOpError()
           << "loop wrapper does not contain exactly one nested op";
 
  Operation &firstOp = *region.op_begin();
  if (!isa<LoopNestOp, LoopWrapperInterface>(firstOp))
    return emitOpError() << "nested in loop wrapper is not another loop "
                            "wrapper or `omp.loop_nest`";
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// ComposableOpInterface
//===----------------------------------------------------------------------===//
 
Operation *ComposableOpInterface::findCapturedOp() {
  Operation *op = this->getOperation();
 
  // Handle the composite case by returning the wrapped omp.loop_nest.
  if (auto wrapperOp = dyn_cast<LoopWrapperInterface>(op))
    return wrapperOp.getWrappedLoop();
 
  // Do not look further if this op is not combined with any of its children.
  // Need to check for composite for the omp.parallel case, which is not a loop
  // wrapper itself.
  if (!isCombined() && !isComposite())
    return op;
 
  Region &region = op->getRegion(0);
  for (Operation &nestedOp : region.getOps()) {
    if (auto wrapperOp = dyn_cast<LoopWrapperInterface>(&nestedOp))
      return wrapperOp.getWrappedLoop();
 
    if (auto composableOp = dyn_cast<ComposableOpInterface>(&nestedOp))
      return composableOp.findCapturedOp();
  }
 
  // This can only be reached if the op has an omp.combined attribute but the
  // corresponding nested composable op has been deleted. In that case, it's
  // correct to return this operation.
  return op;
}
 
LogicalResult ComposableOpInterface::verifyImpl() {
  Operation *op = this->getOperation();
 
  if (op->getNumRegions() != 1)
    return emitOpError() << "composable ops must have a single region";
 
  if (isComposite() && !isa<LoopWrapperInterface, ParallelOp>(op))
    return emitOpError() << "non-loop wrapper cannot be composite";
 
  // If combined, must have exactly one eligible nested op (composable or loop
  // wrapper).
  if (isCombined()) {
    Operation *nestedOp = nullptr;
    auto count = llvm::count_if(
        op->getRegion(0).getOps(), [&nestedOp](mlir::Operation &op) {
          if (isa<ComposableOpInterface, LoopWrapperInterface>(op)) {
            nestedOp = &op;
            return true;
          }
          return false;
        });
 
    // Make an exception for ops marked as omp.combined with no eligible nested
    // ops: this situation should be disallowed, but it can be reached if an
    // MLIR optimization pass find that the child operation has no side effects
    // (many ComposableOpInterface ops have RecursiveMemoryEffects), so it gets
    // deleted without updating the parent's attribute.
    //
    // Since there's a well defined way of handling that situation (treat it as
    // non-combined), we relax the requirement here. Ensuring the parent is
    // updated every time a pass that can potentially remove a child composable
    // op runs is less preferable as a solution.
    if (count == 0)
      return success();
 
    if (count > 1)
      return emitOpError()
             << "multiple eligible child ops found in combined op";
 
    // This operation cannot be combined if its captured nested op can be
    // executed more than once (i.e. its block's successors can reach it) or if
    // it's not guaranteed to be executed before all exits of the region (i.e.
    // it doesn't dominate all blocks with no successors reachable from the
    // entry block).
    DominanceInfo domInfo;
    Block *parentBlock = nestedOp->getBlock();
 
    for (Block *successor : parentBlock->getSuccessors())
      if (successor->isReachable(parentBlock))
        return emitOpError() << "nested combined child op is part of a loop";
 
    for (Block &block : op->getRegion(0))
      if (domInfo.isReachableFromEntry(&block) && block.hasNoSuccessors() &&
          !domInfo.dominates(parentBlock, &block))
        return emitOpError()
               << "nested combined child op doesn't unconditionally execute";
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// LoopOp
//===----------------------------------------------------------------------===//
 
void LoopOp::build(OpBuilder &builder, OperationState &state,
                   const LoopOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
 
  LoopOp::build(builder, state, clauses.bindKind, clauses.privateVars,
                makeArrayAttr(ctx, clauses.privateSyms),
                clauses.privateNeedsBarrier, clauses.order, clauses.orderMod,
                clauses.reductionMod, clauses.reductionVars,
                makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
                makeArrayAttr(ctx, clauses.reductionSyms));
}
 
LogicalResult LoopOp::verify() {
  if (failed(verifyPrivateVarList(*this)))
    return failure();
 
  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}
 
LogicalResult LoopOp::verifyRegions() {
  if (llvm::isa_and_nonnull<LoopWrapperInterface>((*this)->getParentOp()) ||
      getNestedWrapper())
    return emitOpError() << "expected to be a standalone loop wrapper";
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// WsloopOp
//===----------------------------------------------------------------------===//
 
void WsloopOp::build(OpBuilder &builder, OperationState &state,
                     ArrayRef<NamedAttribute> attributes) {
  build(builder, state, /*allocate_vars=*/{}, /*allocator_vars=*/{},
        /*allocate_alignments=*/nullptr,
        /*allocate_private_indices=*/nullptr,
        /*linear_vars=*/ValueRange(), /*linear_step_vars=*/ValueRange(),
        /*linear_var_types*/ nullptr, /*linear_modifiers=*/nullptr,
        /*linear_syms=*/nullptr, /*nowait=*/false, /*order=*/nullptr,
        /*order_mod=*/nullptr,
        /*ordered=*/nullptr, /*private_vars=*/{}, /*private_syms=*/nullptr,
        /*private_needs_barrier=*/false,
        /*reduction_mod=*/nullptr, /*reduction_vars=*/ValueRange(),
        /*reduction_byref=*/nullptr,
        /*reduction_syms=*/nullptr, /*schedule_kind=*/nullptr,
        /*schedule_chunk=*/nullptr, /*schedule_mod=*/nullptr,
        /*schedule_simd=*/false);
  state.addAttributes(attributes);
}
 
void WsloopOp::build(OpBuilder &builder, OperationState &state,
                     const WsloopOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  WsloopOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
      clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
      clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
      clauses.nowait, clauses.order, clauses.orderMod, clauses.ordered,
      clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
      clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
      makeArrayAttr(ctx, clauses.reductionSyms), clauses.scheduleKind,
      clauses.scheduleChunk, clauses.scheduleMod, clauses.scheduleSimd);
}
 
LogicalResult WsloopOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();
 
  if (failed(
          verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars())))
    return failure();
  if (getLinearVars().size() &&
      getLinearVarTypes().value().size() != getLinearVars().size())
    return emitError() << "Ill-formed type attributes for linear variables";
 
  if (failed(verifyLinearSyms(*this)) || failed(verifyPrivateVarList(*this)))
    return failure();
 
  return verifyReductionVarList(*this, getReductionSyms(), getReductionVars(),
                                getReductionByref());
}
 
LogicalResult WsloopOp::verifyRegions() {
  bool isCompositeChildLeaf =
      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
 
  if (LoopWrapperInterface nested = getNestedWrapper()) {
    if (!isComposite())
      return emitError()
             << "'omp.composite' attribute missing from composite wrapper";
 
    // Check for the allowed leaf constructs that may appear in a composite
    // construct directly after DO/FOR.
    if (!isa<SimdOp>(nested))
      return emitError() << "only supported nested wrapper is 'omp.simd'";
 
  } else if (isComposite() && !isCompositeChildLeaf) {
    return emitError()
           << "'omp.composite' attribute present in non-composite wrapper";
  } else if (!isComposite() && isCompositeChildLeaf) {
    return emitError()
           << "'omp.composite' attribute missing from composite wrapper";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Simd construct [2.9.3.1]
//===----------------------------------------------------------------------===//
 
void SimdOp::build(OpBuilder &builder, OperationState &state,
                   const SimdOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  SimdOp::build(
      builder, state, clauses.alignedVars,
      makeArrayAttr(ctx, clauses.alignments), clauses.ifExpr,
      clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
      clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
      clauses.nontemporalVars, clauses.order, clauses.orderMod,
      clauses.privateVars, makeArrayAttr(ctx, clauses.privateSyms),
      clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
      makeArrayAttr(ctx, clauses.reductionSyms), clauses.safelen,
      clauses.simdlen);
}
 
LogicalResult SimdOp::verify() {
  if (getSimdlen().has_value() && getSafelen().has_value() &&
      getSimdlen().value() > getSafelen().value())
    return emitOpError()
           << "simdlen clause and safelen clause are both present, but the "
              "simdlen value is not less than or equal to safelen value";
 
  if (verifyAlignedClause(*this, getAlignments(), getAlignedVars()).failed())
    return failure();
 
  if (verifyNontemporalClause(*this, getNontemporalVars()).failed())
    return failure();
 
  if (failed(
          verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars())))
    return failure();
 
  bool isCompositeChildLeaf =
      llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp());
 
  if (!isComposite() && isCompositeChildLeaf)
    return emitError()
           << "'omp.composite' attribute missing from composite wrapper";
 
  if (isComposite() && !isCompositeChildLeaf)
    return emitError()
           << "'omp.composite' attribute present in non-composite wrapper";
 
  // Firstprivate is not allowed for SIMD in the standard. Check that none of
  // the private decls are for firstprivate.
  std::optional<ArrayAttr> privateSyms = getPrivateSyms();
  if (privateSyms) {
    for (const Attribute &sym : *privateSyms) {
      auto symRef = cast<SymbolRefAttr>(sym);
      omp::PrivateClauseOp privatizer =
          SymbolTable::lookupNearestSymbolFrom<omp::PrivateClauseOp>(
              getOperation(), symRef);
      if (!privatizer)
        return emitError() << "Cannot find privatizer '" << symRef << "'";
      if (privatizer.getDataSharingType() ==
          DataSharingClauseType::FirstPrivate)
        return emitError() << "FIRSTPRIVATE cannot be used with SIMD";
    }
  }
 
  if (failed(verifyPrivateVarList(*this)))
    return failure();
 
  if (getLinearVars().size() &&
      getLinearVarTypes().value().size() != getLinearVars().size())
    return emitError() << "Ill-formed type attributes for linear variables";
 
  if (failed(verifyLinearSyms(*this)))
    return failure();
 
  llvm::DenseSet<Value> privateVars(llvm::from_range, getPrivateVars());
  llvm::DenseSet<Value> reductionVars(llvm::from_range, getReductionVars());
  // TODO Check lastprivate vars when their support is added to SimdOp.
  for (Value var : getLinearVars()) {
    if (privateVars.contains(var) || reductionVars.contains(var))
      return emitOpError()
             << "linear variables cannot appear in other data-sharing clauses";
  }
 
  return success();
}
 
LogicalResult SimdOp::verifyRegions() {
  if (getNestedWrapper())
    return emitOpError() << "must wrap an 'omp.loop_nest' directly";
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Distribute construct [2.9.4.1]
//===----------------------------------------------------------------------===//
 
void DistributeOp::build(OpBuilder &builder, OperationState &state,
                         const DistributeOperands &clauses) {
  DistributeOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(builder.getContext(), clauses.allocateAlignments),
      makeDenseI64ArrayAttr(builder.getContext(),
                            clauses.allocatePrivateIndices),
      clauses.distScheduleStatic, clauses.distScheduleChunkSize, clauses.order,
      clauses.orderMod, clauses.privateVars,
      makeArrayAttr(builder.getContext(), clauses.privateSyms),
      clauses.privateNeedsBarrier);
}
 
LogicalResult DistributeOp::verify() {
  if (this->getDistScheduleChunkSize() && !this->getDistScheduleStatic())
    return emitOpError() << "chunk size set without "
                            "dist_schedule_static being present";
 
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();
 
  if (failed(verifyPrivateVarList(*this)))
    return failure();
 
  return success();
}
 
LogicalResult DistributeOp::verifyRegions() {
  if (LoopWrapperInterface nested = getNestedWrapper()) {
    if (!isComposite())
      return emitError()
             << "'omp.composite' attribute missing from composite wrapper";
    // Check for the allowed leaf constructs that may appear in a composite
    // construct directly after DISTRIBUTE.
    if (isa<WsloopOp>(nested)) {
      Operation *parentOp = (*this)->getParentOp();
      if (!llvm::dyn_cast_if_present<ParallelOp>(parentOp) ||
          !cast<ComposableOpInterface>(parentOp).isComposite()) {
        return emitError() << "an 'omp.wsloop' nested wrapper is only allowed "
                              "when a composite 'omp.parallel' is the direct "
                              "parent";
      }
    } else if (!isa<SimdOp>(nested))
      return emitError() << "only supported nested wrappers are 'omp.simd' and "
                            "'omp.wsloop'";
  } else if (isComposite()) {
    return emitError()
           << "'omp.composite' attribute present in non-composite wrapper";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// DeclareMapperOp / DeclareMapperInfoOp
//===----------------------------------------------------------------------===//
 
void DeclareMapperInfoOp::build(OpBuilder &builder, OperationState &state,
                                const DeclareMapperInfoOperands &clauses) {
  DeclareMapperInfoOp::build(builder, state, clauses.mapVars,
                             clauses.mapIterated);
}
 
LogicalResult DeclareMapperInfoOp::verify() {
  return verifyMapClause(*this, getMapVars(), getMapIterated());
}
 
LogicalResult DeclareMapperOp::verifyRegions() {
  if (!llvm::isa_and_present<DeclareMapperInfoOp>(
          getRegion().getBlocks().front().getTerminator()))
    return emitOpError() << "expected terminator to be a DeclareMapperInfoOp";
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// DeclareReductionOp
//===----------------------------------------------------------------------===//
 
LogicalResult DeclareReductionOp::verifyRegions() {
  if (!getAllocRegion().empty()) {
    for (YieldOp yieldOp : getAllocRegion().getOps<YieldOp>()) {
      if (yieldOp.getResults().size() != 1 ||
          yieldOp.getResults().getTypes()[0] != getType())
        return emitOpError() << "expects alloc region to yield a value "
                                "of the reduction type";
    }
  }
 
  if (getInitializerRegion().empty())
    return emitOpError() << "expects non-empty initializer region";
  Block &initializerEntryBlock = getInitializerRegion().front();
 
  if (initializerEntryBlock.getNumArguments() == 1) {
    if (!getAllocRegion().empty())
      return emitOpError() << "expects two arguments to the initializer region "
                              "when an allocation region is used";
  } else if (initializerEntryBlock.getNumArguments() == 2) {
    if (getAllocRegion().empty())
      return emitOpError() << "expects one argument to the initializer region "
                              "when no allocation region is used";
  } else {
    return emitOpError()
           << "expects one or two arguments to the initializer region";
  }
 
  for (mlir::Value arg : initializerEntryBlock.getArguments())
    if (arg.getType() != getType())
      return emitOpError() << "expects initializer region argument to match "
                              "the reduction type";
 
  for (YieldOp yieldOp : getInitializerRegion().getOps<YieldOp>()) {
    if (yieldOp.getResults().size() != 1 ||
        yieldOp.getResults().getTypes()[0] != getType())
      return emitOpError() << "expects initializer region to yield a value "
                              "of the reduction type";
  }
 
  if (getReductionRegion().empty())
    return emitOpError() << "expects non-empty reduction region";
  Block &reductionEntryBlock = getReductionRegion().front();
  if (reductionEntryBlock.getNumArguments() != 2 ||
      reductionEntryBlock.getArgumentTypes()[0] !=
          reductionEntryBlock.getArgumentTypes()[1] ||
      reductionEntryBlock.getArgumentTypes()[0] != getType())
    return emitOpError() << "expects reduction region with two arguments of "
                            "the reduction type";
  for (YieldOp yieldOp : getReductionRegion().getOps<YieldOp>()) {
    if (yieldOp.getResults().size() != 1 ||
        yieldOp.getResults().getTypes()[0] != getType())
      return emitOpError() << "expects reduction region to yield a value "
                              "of the reduction type";
  }
 
  if (!getAtomicReductionRegion().empty()) {
    Block &atomicReductionEntryBlock = getAtomicReductionRegion().front();
    if (atomicReductionEntryBlock.getNumArguments() != 2 ||
        atomicReductionEntryBlock.getArgumentTypes()[0] !=
            atomicReductionEntryBlock.getArgumentTypes()[1])
      return emitOpError() << "expects atomic reduction region with two "
                              "arguments of the same type";
    auto ptrType = llvm::dyn_cast<PointerLikeType>(
        atomicReductionEntryBlock.getArgumentTypes()[0]);
    if (!ptrType ||
        (ptrType.getElementType() && ptrType.getElementType() != getType()))
      return emitOpError() << "expects atomic reduction region arguments to "
                              "be accumulators containing the reduction type";
  }
 
  if (getCleanupRegion().empty())
    return success();
  Block &cleanupEntryBlock = getCleanupRegion().front();
  if (cleanupEntryBlock.getNumArguments() != 1 ||
      cleanupEntryBlock.getArgument(0).getType() != getType())
    return emitOpError() << "expects cleanup region with one argument "
                            "of the reduction type";
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// TaskOp
//===----------------------------------------------------------------------===//
 
void TaskOp::build(OpBuilder &builder, OperationState &state,
                   const TaskOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TaskOp::build(builder, state, clauses.iterated, clauses.affinityVars,
                clauses.allocateVars, clauses.allocatorVars,
                makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                makeArrayAttr(ctx, clauses.dependKinds), clauses.dependVars,
                makeArrayAttr(ctx, clauses.dependIteratedKinds),
                clauses.dependIterated, clauses.final, clauses.ifExpr,
                clauses.inReductionVars,
                makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
                makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
                clauses.priority, /*private_vars=*/clauses.privateVars,
                /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),
                clauses.privateNeedsBarrier, clauses.threadset, clauses.untied,
                clauses.eventHandle);
}
 
LogicalResult TaskOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();
 
  LogicalResult verifyDependVars =
      verifyDependVarList(*this, getDependKinds(), getDependVars(),
                          getDependIteratedKinds(), getDependIterated());
  if (failed(verifyDependVars))
    return verifyDependVars;
 
  if (failed(verifyPrivateVarList(*this)))
    return failure();
 
  return verifyReductionVarList(*this, getInReductionSyms(),
                                getInReductionVars(), getInReductionByref());
}
 
//===----------------------------------------------------------------------===//
// TaskgroupOp
//===----------------------------------------------------------------------===//
 
void TaskgroupOp::build(OpBuilder &builder, OperationState &state,
                        const TaskgroupOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TaskgroupOp::build(builder, state, clauses.allocateVars,
                     clauses.allocatorVars,
                     makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
                     makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices),
                     clauses.taskReductionVars,
                     makeDenseBoolArrayAttr(ctx, clauses.taskReductionByref),
                     makeArrayAttr(ctx, clauses.taskReductionSyms));
}
 
LogicalResult TaskgroupOp::verify() {
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr())))
    return failure();
 
  return verifyReductionVarList(*this, getTaskReductionSyms(),
                                getTaskReductionVars(),
                                getTaskReductionByref());
}
 
//===----------------------------------------------------------------------===//
// TaskloopContextOp
//===----------------------------------------------------------------------===//
 
void TaskloopContextOp::build(OpBuilder &builder, OperationState &state,
                              const TaskloopContextOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  TaskloopContextOp::build(
      builder, state, clauses.allocateVars, clauses.allocatorVars,
      makeDenseI64ArrayAttr(ctx, clauses.allocateAlignments),
      makeDenseI64ArrayAttr(ctx, clauses.allocatePrivateIndices), clauses.final,
      clauses.grainsizeMod, clauses.grainsize, clauses.ifExpr,
      clauses.inReductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.inReductionByref),
      makeArrayAttr(ctx, clauses.inReductionSyms), clauses.mergeable,
      clauses.nogroup, clauses.numTasksMod, clauses.numTasks, clauses.priority,
      /*private_vars=*/clauses.privateVars,
      /*private_syms=*/makeArrayAttr(ctx, clauses.privateSyms),
      clauses.privateNeedsBarrier, clauses.reductionMod, clauses.reductionVars,
      makeDenseBoolArrayAttr(ctx, clauses.reductionByref),
      makeArrayAttr(ctx, clauses.reductionSyms), clauses.threadset,
      clauses.untied);
  state.addAttribute("omp.combined", UnitAttr::get(ctx));
}
 
TaskloopWrapperOp TaskloopContextOp::getLoopOp() {
  return cast<TaskloopWrapperOp>(
      *llvm::find_if(getRegion().front(), [](mlir::Operation &op) {
        return isa<TaskloopWrapperOp>(op);
      }));
}
 
LogicalResult TaskloopContextOp::verify() {
  if (failed(verifyPrivateVarList(*this)))
    return failure();
  if (failed(verifyAllocateClause(
          getOperation(), getAllocateVars(), getAllocatorVars(),
          getAllocateAlignmentsAttr(), getAllocatePrivateIndicesAttr(),
          getPrivateVars(), getPrivateSymsAttr())))
    return failure();
 
  if (failed(verifyReductionVarList(*this, getReductionSyms(),
                                    getReductionVars(), getReductionByref())) ||
      failed(verifyReductionVarList(*this, getInReductionSyms(),
                                    getInReductionVars(),
                                    getInReductionByref())))
    return failure();
 
  if (!getReductionVars().empty() && getNogroup())
    return emitError("if a reduction clause is present on the taskloop "
                     "directive, the nogroup clause must not be specified");
  for (auto var : getReductionVars()) {
    if (llvm::is_contained(getInReductionVars(), var))
      return emitError("the same list item cannot appear in both a reduction "
                       "and an in_reduction clause");
  }
 
  if (getGrainsize() && getNumTasks()) {
    return emitError(
        "the grainsize clause and num_tasks clause are mutually exclusive and "
        "may not appear on the same taskloop directive");
  }
 
  // Without this restriction, any compound construct including `taskloop` would
  // fail to correctly identify the whole chain of operations (see
  // ComposableOpInterface::findCapturedOp()), as well as failing to do so even
  // for standalone `taskloop` constructs.
  if (!isCombined())
    return emitOpError("must always contain the 'omp.combined' attribute");
 
  return success();
}
 
LogicalResult TaskloopContextOp::verifyRegions() {
  Region &region = getRegion();
  auto loopWrapperIt = llvm::find_if(region.front(), [](mlir::Operation &op) {
    return isa<TaskloopWrapperOp>(op);
  });
  if (loopWrapperIt == region.front().end())
    return emitOpError()
           << "expected a TaskloopWrapperOp directly nested in the region";
 
  auto loopWrapperOp = cast<TaskloopWrapperOp>(*loopWrapperIt);
  auto loopNestOp = dyn_cast<LoopNestOp>(loopWrapperOp.getWrappedLoop());
  // This will fail the verifier for TaskloopWrapperOp and print an error
  // message there.
  if (!loopNestOp)
    return failure();
 
  std::function<bool(Value)> isValidBoundValue = [&](Value value) -> bool {
    Region *valueRegion = value.getParentRegion();
    // A loop bound value defined outside of the taskloop context region is
    // valid. A region is considered an ancestor of itself.
    if (!region.isAncestor(valueRegion))
      return true;
 
    Operation *defOp = value.getDefiningOp();
    if (!defOp || defOp->getNumRegions() != 0 || !isPure(defOp))
      return false;
 
    return llvm::all_of(defOp->getOperands(), isValidBoundValue);
  };
  auto hasUnsupportedTaskloopLocalBound = [&](OperandRange range) -> bool {
    return llvm::any_of(range,
                        [&](Value value) { return !isValidBoundValue(value); });
  };
 
  if (hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopLowerBounds()) ||
      hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopUpperBounds()) ||
      hasUnsupportedTaskloopLocalBound(loopNestOp.getLoopSteps())) {
    return emitOpError()
           << "expects loop bounds and steps to be defined outside of the "
              "taskloop.context region or by pure, regionless operations "
              "that do not depend on block arguments";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// TaskloopWrapperOp
//===----------------------------------------------------------------------===//
 
void TaskloopWrapperOp::build(OpBuilder &builder, OperationState &state,
                              const TaskloopWrapperOperands &clauses) {
  TaskloopWrapperOp::build(builder, state);
}
 
TaskloopContextOp TaskloopWrapperOp::getTaskloopContext() {
  return dyn_cast<TaskloopContextOp>(getOperation()->getParentOp());
}
 
LogicalResult TaskloopWrapperOp::verify() {
  TaskloopContextOp context = getTaskloopContext();
  if (!context)
    return emitOpError() << "expected to be nested in a taskloop context op";
  return success();
}
 
LogicalResult TaskloopWrapperOp::verifyRegions() {
  if (LoopWrapperInterface nested = getNestedWrapper()) {
    if (!isComposite())
      return emitError()
             << "'omp.composite' attribute missing from composite wrapper";
 
    // Check for the allowed leaf constructs that may appear in a composite
    // construct directly after TASKLOOP.
    if (!isa<SimdOp>(nested))
      return emitError() << "only supported nested wrapper is 'omp.simd'";
  } else if (isComposite()) {
    return emitError()
           << "'omp.composite' attribute present in non-composite wrapper";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// LoopNestOp
//===----------------------------------------------------------------------===//
 
ParseResult LoopNestOp::parse(OpAsmParser &parser, OperationState &result) {
  // Parse an opening `(` followed by induction variables followed by `)`
  SmallVector<OpAsmParser::Argument> ivs;
  SmallVector<OpAsmParser::UnresolvedOperand> lbs, ubs;
  Type loopVarType;
  if (parser.parseArgumentList(ivs, OpAsmParser::Delimiter::Paren) ||
      parser.parseColonType(loopVarType) ||
      // Parse loop bounds.
      parser.parseEqual() ||
      parser.parseOperandList(lbs, ivs.size(), OpAsmParser::Delimiter::Paren) ||
      parser.parseKeyword("to") ||
      parser.parseOperandList(ubs, ivs.size(), OpAsmParser::Delimiter::Paren))
    return failure();
 
  for (auto &iv : ivs)
    iv.type = loopVarType;
 
  auto *ctx = parser.getBuilder().getContext();
  // Parse "inclusive" flag.
  if (succeeded(parser.parseOptionalKeyword("inclusive")))
    result.addAttribute("loop_inclusive", UnitAttr::get(ctx));
 
  // Parse step values.
  SmallVector<OpAsmParser::UnresolvedOperand> steps;
  if (parser.parseKeyword("step") ||
      parser.parseOperandList(steps, ivs.size(), OpAsmParser::Delimiter::Paren))
    return failure();
 
  // Parse collapse
  int64_t value = 0;
  if (!parser.parseOptionalKeyword("collapse") &&
      (parser.parseLParen() || parser.parseInteger(value) ||
       parser.parseRParen()))
    return failure();
  if (value > 1)
    result.addAttribute(
        "collapse_num_loops",
        IntegerAttr::get(parser.getBuilder().getI64Type(), value));
 
  // Parse tiles
  SmallVector<int64_t> tiles;
  auto parseTiles = [&]() -> ParseResult {
    int64_t tile;
    if (parser.parseInteger(tile))
      return failure();
    tiles.push_back(tile);
    return success();
  };
 
  if (!parser.parseOptionalKeyword("tiles") &&
      (parser.parseLParen() || parser.parseCommaSeparatedList(parseTiles) ||
       parser.parseRParen()))
    return failure();
 
  if (tiles.size() > 0)
    result.addAttribute("tile_sizes", DenseI64ArrayAttr::get(ctx, tiles));
 
  // Parse the body.
  Region *region = result.addRegion();
  if (parser.parseRegion(*region, ivs))
    return failure();
 
  // Resolve operands.
  if (parser.resolveOperands(lbs, loopVarType, result.operands) ||
      parser.resolveOperands(ubs, loopVarType, result.operands) ||
      parser.resolveOperands(steps, loopVarType, result.operands))
    return failure();
 
  // Parse the optional attribute list.
  return parser.parseOptionalAttrDict(result.attributes);
}
 
void LoopNestOp::print(OpAsmPrinter &p) {
  Region &region = getRegion();
  auto args = region.getArguments();
  p << " (" << args << ") : " << args[0].getType() << " = ("
    << getLoopLowerBounds() << ") to (" << getLoopUpperBounds() << ") ";
  if (getLoopInclusive())
    p << "inclusive ";
  p << "step (" << getLoopSteps() << ") ";
  if (int64_t numCollapse = getCollapseNumLoops())
    if (numCollapse > 1)
      p << "collapse(" << numCollapse << ") ";
 
  if (const auto tiles = getTileSizes())
    p << "tiles(" << tiles.value() << ") ";
 
  p.printRegion(region, /*printEntryBlockArgs=*/false);
}
 
void LoopNestOp::build(OpBuilder &builder, OperationState &state,
                       const LoopNestOperands &clauses) {
  MLIRContext *ctx = builder.getContext();
  LoopNestOp::build(builder, state, clauses.collapseNumLoops,
                    clauses.loopLowerBounds, clauses.loopUpperBounds,
                    clauses.loopSteps, clauses.loopInclusive,
                    makeDenseI64ArrayAttr(ctx, clauses.tileSizes));
}
 
LogicalResult LoopNestOp::verify() {
  if (getLoopLowerBounds().empty())
    return emitOpError() << "must represent at least one loop";
 
  if (getLoopLowerBounds().size() != getIVs().size())
    return emitOpError() << "number of range arguments and IVs do not match";
 
  for (auto [lb, iv] : llvm::zip_equal(getLoopLowerBounds(), getIVs())) {
    if (lb.getType() != iv.getType())
      return emitOpError()
             << "range argument type does not match corresponding IV type";
  }
 
  uint64_t numIVs = getIVs().size();
 
  if (const auto &numCollapse = getCollapseNumLoops())
    if (numCollapse > numIVs)
      return emitOpError()
             << "collapse value is larger than the number of loops";
 
  if (const auto &tiles = getTileSizes())
    if (tiles.value().size() > numIVs)
      return emitOpError() << "too few canonical loops for tile dimensions";
 
  if (!llvm::dyn_cast_if_present<LoopWrapperInterface>((*this)->getParentOp()))
    return emitOpError() << "expects parent op to be a loop wrapper";
 
  return success();
}
 
void LoopNestOp::gatherWrappers(
    SmallVectorImpl<LoopWrapperInterface> &wrappers) {
  Operation *parent = (*this)->getParentOp();
  while (auto wrapper =
             llvm::dyn_cast_if_present<LoopWrapperInterface>(parent)) {
    wrappers.push_back(wrapper);
    parent = parent->getParentOp();
  }
}
 
//===----------------------------------------------------------------------===//
// OpenMP canonical loop handling
//===----------------------------------------------------------------------===//
 
std::tuple<NewCliOp, OpOperand *, OpOperand *>
mlir::omp ::decodeCli(Value cli) {
 
  // Defining a CLI for a generated loop is optional; if there is none then
  // there is no followup-tranformation
  if (!cli)
    return {{}, nullptr, nullptr};
 
  assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&
         "Unexpected type of cli");
 
  NewCliOp create = cast<NewCliOp>(cli.getDefiningOp());
  OpOperand *gen = nullptr;
  OpOperand *cons = nullptr;
  for (OpOperand &use : cli.getUses()) {
    auto op = cast<LoopTransformationInterface>(use.getOwner());
 
    unsigned opnum = use.getOperandNumber();
    if (op.isGeneratee(opnum)) {
      assert(!gen && "Each CLI may have at most one def");
      gen = &use;
    } else if (op.isApplyee(opnum)) {
      assert(!cons && "Each CLI may have at most one consumer");
      cons = &use;
    } else {
      llvm_unreachable("Unexpected operand for a CLI");
    }
  }
 
  return {create, gen, cons};
}
 
ClauseProcBindKind
mlir::omp::convertProcBindKind(llvm::omp::ProcBindKind kind) {
  switch (kind) {
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_close:
    return ClauseProcBindKind::Close;
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_master:
    return ClauseProcBindKind::Master;
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_primary:
    return ClauseProcBindKind::Primary;
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_spread:
    return ClauseProcBindKind::Spread;
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_default:
  case llvm::omp::ProcBindKind::OMP_PROC_BIND_unknown:
    break;
  }
  llvm_unreachable("unexpected proc-bind kind");
}
 
void NewCliOp::build(::mlir::OpBuilder &odsBuilder,
                     ::mlir::OperationState &odsState) {
  odsState.addTypes(CanonicalLoopInfoType::get(odsBuilder.getContext()));
}
 
void NewCliOp::getAsmResultNames(OpAsmSetValueNameFn setNameFn) {
  Value result = getResult();
  auto [newCli, gen, cons] = decodeCli(result);
 
  // Structured binding `gen` cannot be captured in lambdas before C++20
  OpOperand *generator = gen;
 
  // Derive the CLI variable name from its generator:
  //  * "canonloop" for omp.canonical_loop
  //  * custom name for loop transformation generatees
  //  * "cli" as fallback if no generator
  //  * "_r<idx>" suffix for nested loops, where <idx> is the sequential order
  //  at that level
  //  * "_s<idx>" suffix for operations with multiple regions, where <idx> is
  //  the index of that region
  std::string cliName{"cli"};
  if (gen) {
    cliName =
        TypeSwitch<Operation *, std::string>(gen->getOwner())
            .Case([&](CanonicalLoopOp op) {
              return generateLoopNestingName("canonloop", op);
            })
            .Case([&](UnrollHeuristicOp op) -> std::string {
              llvm_unreachable("heuristic unrolling does not generate a loop");
            })
            .Case([&](FuseOp op) -> std::string {
              unsigned opnum = generator->getOperandNumber();
              // The position of the first loop to be fused is the same position
              // as the resulting fused loop
              if (op.getFirst().has_value() && opnum != op.getFirst().value())
                return "canonloop_fuse";
              else
                return "fused";
            })
            .Case([&](TileOp op) -> std::string {
              auto [generateesFirst, generateesCount] =
                  op.getGenerateesODSOperandIndexAndLength();
              unsigned firstGrid = generateesFirst;
              unsigned firstIntratile = generateesFirst + generateesCount / 2;
              unsigned end = generateesFirst + generateesCount;
              unsigned opnum = generator->getOperandNumber();
              // In the OpenMP apply and looprange clauses, indices are 1-based
              if (firstGrid <= opnum && opnum < firstIntratile) {
                unsigned gridnum = opnum - firstGrid + 1;
                return ("grid" + Twine(gridnum)).str();
              }
              if (firstIntratile <= opnum && opnum < end) {
                unsigned intratilenum = opnum - firstIntratile + 1;
                return ("intratile" + Twine(intratilenum)).str();
              }
              llvm_unreachable("Unexpected generatee argument");
            })
            .DefaultUnreachable("TODO: Custom name for this operation");
  }
 
  setNameFn(result, cliName);
}
 
LogicalResult NewCliOp::verify() {
  Value cli = getResult();
 
  assert(cli.getType() == CanonicalLoopInfoType::get(cli.getContext()) &&
         "Unexpected type of cli");
 
  // Check that the CLI is used in at most generator and one consumer
  OpOperand *gen = nullptr;
  OpOperand *cons = nullptr;
  for (mlir::OpOperand &use : cli.getUses()) {
    auto op = cast<mlir::omp::LoopTransformationInterface>(use.getOwner());
 
    unsigned opnum = use.getOperandNumber();
    if (op.isGeneratee(opnum)) {
      if (gen) {
        InFlightDiagnostic error =
            emitOpError("CLI must have at most one generator");
        error.attachNote(gen->getOwner()->getLoc())
            .append("first generator here:");
        error.attachNote(use.getOwner()->getLoc())
            .append("second generator here:");
        return error;
      }
 
      gen = &use;
    } else if (op.isApplyee(opnum)) {
      if (cons) {
        InFlightDiagnostic error =
            emitOpError("CLI must have at most one consumer");
        error.attachNote(cons->getOwner()->getLoc())
            .append("first consumer here:")
            .appendOp(*cons->getOwner(),
                      OpPrintingFlags().printGenericOpForm());
        error.attachNote(use.getOwner()->getLoc())
            .append("second consumer here:")
            .appendOp(*use.getOwner(), OpPrintingFlags().printGenericOpForm());
        return error;
      }
 
      cons = &use;
    } else {
      llvm_unreachable("Unexpected operand for a CLI");
    }
  }
 
  // If the CLI is source of a transformation, it must have a generator
  if (cons && !gen) {
    InFlightDiagnostic error = emitOpError("CLI has no generator");
    error.attachNote(cons->getOwner()->getLoc())
        .append("see consumer here: ")
        .appendOp(*cons->getOwner(), OpPrintingFlags().printGenericOpForm());
    return error;
  }
 
  return success();
}
 
void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                            Value tripCount) {
  odsState.addOperands(tripCount);
  odsState.addOperands(Value());
  (void)odsState.addRegion();
}
 
void CanonicalLoopOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                            Value tripCount, ::mlir::Value cli) {
  odsState.addOperands(tripCount);
  odsState.addOperands(cli);
  (void)odsState.addRegion();
}
 
void CanonicalLoopOp::getAsmBlockNames(OpAsmSetBlockNameFn setNameFn) {
  setNameFn(&getRegion().front(), "body_entry");
}
 
void CanonicalLoopOp::getAsmBlockArgumentNames(Region &region,
                                               OpAsmSetValueNameFn setNameFn) {
  std::string ivName = generateLoopNestingName("iv", *this);
  setNameFn(region.getArgument(0), ivName);
}
 
void CanonicalLoopOp::print(OpAsmPrinter &p) {
  if (getCli())
    p << '(' << getCli() << ')';
  p << ' ' << getInductionVar() << " : " << getInductionVar().getType()
    << " in range(" << getTripCount() << ") ";
 
  p.printRegion(getRegion(), /*printEntryBlockArgs=*/false,
                /*printBlockTerminators=*/true);
 
  p.printOptionalAttrDict((*this)->getDiscardableAttrDictionary().getValue());
}
 
mlir::ParseResult CanonicalLoopOp::parse(::mlir::OpAsmParser &parser,
                                         ::mlir::OperationState &result) {
  CanonicalLoopInfoType cliType =
      CanonicalLoopInfoType::get(parser.getContext());
 
  // Parse (optional) omp.cli identifier
  OpAsmParser::UnresolvedOperand cli;
  SmallVector<mlir::Value, 1> cliOperand;
  if (!parser.parseOptionalLParen()) {
    if (parser.parseOperand(cli) ||
        parser.resolveOperand(cli, cliType, cliOperand) || parser.parseRParen())
      return failure();
  }
 
  // We derive the type of tripCount from inductionVariable. MLIR requires the
  // type of tripCount to be known when calling resolveOperand so we have parse
  // the type before processing the inductionVariable.
  OpAsmParser::Argument inductionVariable;
  OpAsmParser::UnresolvedOperand tripcount;
  if (parser.parseArgument(inductionVariable, /*allowType*/ true) ||
      parser.parseKeyword("in") || parser.parseKeyword("range") ||
      parser.parseLParen() || parser.parseOperand(tripcount) ||
      parser.parseRParen() ||
      parser.resolveOperand(tripcount, inductionVariable.type, result.operands))
    return failure();
 
  // Parse the loop body.
  Region *region = result.addRegion();
  if (parser.parseRegion(*region, {inductionVariable}))
    return failure();
 
  // We parsed the cli operand forst, but because it is optional, it must be
  // last in the operand list.
  result.operands.append(cliOperand);
 
  // Parse the optional attribute list.
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();
 
  return mlir::success();
}
 
LogicalResult CanonicalLoopOp::verify() {
  // The region's entry must accept the induction variable
  // It can also be empty if just created
  if (!getRegion().empty()) {
    Region &region = getRegion();
    if (region.getNumArguments() != 1)
      return emitOpError(
          "Canonical loop region must have exactly one argument");
 
    if (getInductionVar().getType() != getTripCount().getType())
      return emitOpError(
          "Region argument must be the same type as the trip count");
  }
 
  return success();
}
 
Value CanonicalLoopOp::getInductionVar() { return getRegion().getArgument(0); }
 
std::pair<unsigned, unsigned>
CanonicalLoopOp::getApplyeesODSOperandIndexAndLength() {
  // No applyees
  return {0, 0};
}
 
std::pair<unsigned, unsigned>
CanonicalLoopOp::getGenerateesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_cli);
}
 
//===----------------------------------------------------------------------===//
// UnrollHeuristicOp
//===----------------------------------------------------------------------===//
 
void UnrollHeuristicOp::build(::mlir::OpBuilder &odsBuilder,
                              ::mlir::OperationState &odsState,
                              ::mlir::Value cli) {
  odsState.addOperands(cli);
}
 
void UnrollHeuristicOp::print(OpAsmPrinter &p) {
  p << '(' << getApplyee() << ')';
 
  p.printOptionalAttrDict((*this)->getDiscardableAttrDictionary().getValue());
}
 
mlir::ParseResult UnrollHeuristicOp::parse(::mlir::OpAsmParser &parser,
                                           ::mlir::OperationState &result) {
  auto cliType = CanonicalLoopInfoType::get(parser.getContext());
 
  if (parser.parseLParen())
    return failure();
 
  OpAsmParser::UnresolvedOperand applyee;
  if (parser.parseOperand(applyee) ||
      parser.resolveOperand(applyee, cliType, result.operands))
    return failure();
 
  if (parser.parseRParen())
    return failure();
 
  // Optional output loop (full unrolling has none)
  if (!parser.parseOptionalArrow()) {
    if (parser.parseLParen() || parser.parseRParen())
      return failure();
  }
 
  // Parse the optional attribute list.
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();
 
  return mlir::success();
}
 
std::pair<unsigned, unsigned>
UnrollHeuristicOp ::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyee);
}
 
std::pair<unsigned, unsigned>
UnrollHeuristicOp::getGenerateesODSOperandIndexAndLength() {
  return {0, 0};
}
 
//===----------------------------------------------------------------------===//
// UnrollFullOp
//===----------------------------------------------------------------------===//
 
void UnrollFullOp::build(::mlir::OpBuilder &odsBuilder,
                         ::mlir::OperationState &odsState, ::mlir::Value cli) {
  odsState.addOperands(cli);
}
 
void UnrollFullOp::print(OpAsmPrinter &p) {
  p << '(' << getApplyee() << ')';
 
  p.printOptionalAttrDict((*this)->getDiscardableAttrDictionary().getValue());
}
 
mlir::ParseResult UnrollFullOp::parse(::mlir::OpAsmParser &parser,
                                      ::mlir::OperationState &result) {
  auto cliType = CanonicalLoopInfoType::get(parser.getContext());
 
  if (parser.parseLParen())
    return failure();
 
  OpAsmParser::UnresolvedOperand applyee;
  if (parser.parseOperand(applyee) ||
      parser.resolveOperand(applyee, cliType, result.operands))
    return failure();
 
  if (parser.parseRParen())
    return failure();
 
  // Optional output loop; full unrolling has none.
  if (!parser.parseOptionalArrow()) {
    if (parser.parseLParen() || parser.parseRParen())
      return failure();
  }
 
  // Parse the optional attribute list.
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();
 
  return mlir::success();
}
 
std::pair<unsigned, unsigned>
UnrollFullOp::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyee);
}
 
std::pair<unsigned, unsigned>
UnrollFullOp::getGenerateesODSOperandIndexAndLength() {
  return {0, 0};
}
 
LogicalResult UnrollFullOp::verify() {
  auto [create, gen, cons] = decodeCli(getApplyee());
  if (!gen)
    return emitOpError() << "applyee CLI has no generator";
 
  // Full unrolling leaves no loop, so the trip count must be constant. Only
  // omp.canonical_loop states one.
  if (auto loop = dyn_cast<CanonicalLoopOp>(gen->getOwner())) {
    if (!matchPattern(loop.getTripCount(), m_Constant()))
      return emitOpError() << "applyee loop must have a constant trip count";
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// UnrollPartialOp
//===----------------------------------------------------------------------===//
 
void UnrollPartialOp::build(::mlir::OpBuilder &odsBuilder,
                            ::mlir::OperationState &odsState, ::mlir::Value cli,
                            uint64_t unrollFactor) {
  odsState.addOperands(cli);
  Properties &props = odsState.getOrAddProperties<Properties>();
  props.unroll_factor = odsBuilder.getI64IntegerAttr(unrollFactor);
}
 
void UnrollPartialOp::print(OpAsmPrinter &p) {
  p << '(' << getApplyee() << ')';
 
  SmallVector<NamedAttribute> attrs((*this)->getDiscardableAttrs());
  attrs.emplace_back(getUnrollFactorAttrName(), getUnrollFactorAttr());
  llvm::sort(attrs);
  p.printOptionalAttrDict(attrs);
}
 
mlir::ParseResult UnrollPartialOp::parse(::mlir::OpAsmParser &parser,
                                         ::mlir::OperationState &result) {
  auto cliType = CanonicalLoopInfoType::get(parser.getContext());
 
  if (parser.parseLParen())
    return failure();
 
  OpAsmParser::UnresolvedOperand applyee;
  if (parser.parseOperand(applyee) ||
      parser.resolveOperand(applyee, cliType, result.operands))
    return failure();
 
  if (parser.parseRParen())
    return failure();
 
  // The unroll factor is carried by the `unroll_factor` attribute.
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();
 
  return mlir::success();
}
 
std::pair<unsigned, unsigned>
UnrollPartialOp::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyee);
}
 
std::pair<unsigned, unsigned>
UnrollPartialOp::getGenerateesODSOperandIndexAndLength() {
  return {0, 0};
}
 
//===----------------------------------------------------------------------===//
// TileOp
//===----------------------------------------------------------------------===//
 
static void printLoopTransformClis(OpAsmPrinter &p, TileOp op,
                                   OperandRange generatees,
                                   OperandRange applyees) {
  if (!generatees.empty())
    p << '(' << llvm::interleaved(generatees) << ')';
 
  if (!applyees.empty())
    p << " <- (" << llvm::interleaved(applyees) << ')';
}
 
static ParseResult parseLoopTransformClis(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &generateesOperands,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &applyeesOperands) {
  if (parser.parseOptionalLess()) {
    // Syntax 1: generatees present
 
    if (parser.parseOperandList(generateesOperands,
                                mlir::OpAsmParser::Delimiter::Paren))
      return failure();
 
    if (parser.parseLess())
      return failure();
  } else {
    // Syntax 2: generatees omitted
  }
 
  // Parse `<-` (`<` has already been parsed)
  if (parser.parseMinus())
    return failure();
 
  if (parser.parseOperandList(applyeesOperands,
                              mlir::OpAsmParser::Delimiter::Paren))
    return failure();
 
  return success();
}
 
/// Check properties of the loop nest consisting of the transformation's
/// applyees:
/// 1. They are nested inside each other
/// 2. They are perfectly nested
///    (no code with side-effects in-between the loops)
/// 3. They are rectangular
///    (loop bounds are invariant in respect to the outer loops)
///
/// TODO: Generalize for LoopTransformationInterface.
static LogicalResult checkApplyeesNesting(TileOp op) {
  // Collect the loops from the nest
  bool isOnlyCanonLoops = true;
  SmallVector<CanonicalLoopOp> canonLoops;
  for (Value applyee : op.getApplyees()) {
    auto [create, gen, cons] = decodeCli(applyee);
 
    if (!gen)
      return op.emitOpError() << "applyee CLI has no generator";
 
    auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());
    canonLoops.push_back(loop);
    if (!loop)
      isOnlyCanonLoops = false;
  }
 
  // FIXME: We currently can only verify non-rectangularity and perfect nest of
  // omp.canonical_loop.
  if (!isOnlyCanonLoops)
    return success();
 
  DenseSet<Value> parentIVs;
  for (auto i : llvm::seq<int>(1, canonLoops.size())) {
    auto parentLoop = canonLoops[i - 1];
    auto loop = canonLoops[i];
 
    if (parentLoop.getOperation() != loop.getOperation()->getParentOp())
      return op.emitOpError()
             << "tiled loop nest must be nested within each other";
 
    parentIVs.insert(parentLoop.getInductionVar());
 
    // Canonical loop must be perfectly nested, i.e. the body of the parent must
    // only contain the omp.canonical_loop of the nested loops, and
    // omp.terminator
    bool isPerfectlyNested = [&]() {
      auto &parentBody = parentLoop.getRegion();
      if (!parentBody.hasOneBlock())
        return false;
      auto &parentBlock = parentBody.getBlocks().front();
 
      auto nestedLoopIt = parentBlock.begin();
      if (nestedLoopIt == parentBlock.end() ||
          (&*nestedLoopIt != loop.getOperation()))
        return false;
 
      auto termIt = std::next(nestedLoopIt);
      if (termIt == parentBlock.end() || !isa<TerminatorOp>(termIt))
        return false;
 
      if (std::next(termIt) != parentBlock.end())
        return false;
 
      return true;
    }();
    if (!isPerfectlyNested)
      return op.emitOpError() << "tiled loop nest must be perfectly nested";
 
    if (parentIVs.contains(loop.getTripCount()))
      return op.emitOpError() << "tiled loop nest must be rectangular";
  }
 
  // TODO: The tile sizes must be computed before the loop, but checking this
  // requires dominance analysis. For instance:
  //
  //      %canonloop = omp.new_cli
  //      omp.canonical_loop(%canonloop) %iv : i32 in range(%tc) {
  //        // write to %x
  //        omp.terminator
  //      }
  //      %ts = llvm.load %x
  //      omp.tile <- (%canonloop) sizes(%ts : i32)
 
  return success();
}
 
LogicalResult TileOp::verify() {
  if (getApplyees().empty())
    return emitOpError() << "must apply to at least one loop";
 
  if (getSizes().size() != getApplyees().size())
    return emitOpError() << "there must be one tile size for each applyee";
 
  if (!getGeneratees().empty() &&
      2 * getSizes().size() != getGeneratees().size())
    return emitOpError()
           << "expecting two times the number of generatees than applyees";
 
  return checkApplyeesNesting(*this);
}
 
std::pair<unsigned, unsigned> TileOp ::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyees);
}
 
std::pair<unsigned, unsigned> TileOp::getGenerateesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_generatees);
}
 
//===----------------------------------------------------------------------===//
// FuseOp
//===----------------------------------------------------------------------===//
 
static void printLoopTransformClis(OpAsmPrinter &p, FuseOp op,
                                   OperandRange generatees,
                                   OperandRange applyees) {
  if (!generatees.empty())
    p << '(' << llvm::interleaved(generatees) << ')';
 
  if (!applyees.empty())
    p << " <- (" << llvm::interleaved(applyees) << ')';
}
 
LogicalResult FuseOp::verify() {
  if (getApplyees().size() < 2)
    return emitOpError() << "must apply to at least two loops";
 
  if (getFirst().has_value() && getCount().has_value()) {
    int64_t first = getFirst().value();
    int64_t count = getCount().value();
    if ((unsigned)(first + count - 1) > getApplyees().size())
      return emitOpError() << "the numbers of applyees must be at least first "
                              "minus one plus count attributes";
    if (!getGeneratees().empty() &&
        getGeneratees().size() != getApplyees().size() + 1 - count)
      return emitOpError() << "the number of generatees must be the number of "
                              "aplyees plus one minus count";
 
  } else {
    if (!getGeneratees().empty() && getGeneratees().size() != 1)
      return emitOpError()
             << "in a complete fuse the number of generatees must be exactly 1";
  }
  for (auto &&applyee : getApplyees()) {
    auto [create, gen, cons] = decodeCli(applyee);
 
    if (!gen)
      return emitOpError() << "applyee CLI has no generator";
    auto loop = dyn_cast_or_null<CanonicalLoopOp>(gen->getOwner());
    if (!loop)
      return emitOpError()
             << "currently only supports omp.canonical_loop as applyee";
  }
  return success();
}
std::pair<unsigned, unsigned> FuseOp::getApplyeesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_applyees);
}
 
std::pair<unsigned, unsigned> FuseOp::getGenerateesODSOperandIndexAndLength() {
  return getODSOperandIndexAndLength(odsIndex_generatees);
}
 
//===----------------------------------------------------------------------===//
// Critical construct (2.17.1)
//===----------------------------------------------------------------------===//
 
void CriticalDeclareOp::build(OpBuilder &builder, OperationState &state,
                              const CriticalDeclareOperands &clauses) {
  CriticalDeclareOp::build(builder, state, clauses.symName,
                           clauses.symVisibility, clauses.hint);
}
 
LogicalResult CriticalDeclareOp::verify() {
  return verifySynchronizationHint(*this, getHint());
}
 
LogicalResult CriticalOp::verify() {
  SymbolRefAttr currentName = getNameAttr();
 
  CriticalOp parentCritical = (*this)->getParentOfType<CriticalOp>();
 
  while (parentCritical) {
    SymbolRefAttr parentName = parentCritical.getNameAttr();
 
    if (currentName == parentName) {
      if (currentName) {
        return emitOpError() << "cannot be nested inside another omp.critical "
                                "region with the same name ("
                             << currentName << ")";
      } else {
        return emitOpError() << "cannot be nested inside another unnamed "
                                "omp.critical region";
      }
    }
 
    parentCritical = parentCritical->getParentOfType<CriticalOp>();
  }
 
  return success();
}
 
LogicalResult CriticalOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  if (getNameAttr()) {
    SymbolRefAttr symbolRef = getNameAttr();
    auto decl = symbolTable.lookupNearestSymbolFrom<CriticalDeclareOp>(
        *this, symbolRef);
    if (!decl) {
      return emitOpError() << "expected symbol reference " << symbolRef
                           << " to point to a critical declaration";
    }
  }
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Spec 5.1: Error directive (2.5.4)
//===----------------------------------------------------------------------===//
 
LogicalResult ErrorOp::verify() {
  if (getMessage() && getMessageExpr())
    return emitOpError() << "the message must be provided either as a constant "
                            "`message` attribute or as a `message_expr` "
                            "operand, but not both";
  return success();
}
 
//===----------------------------------------------------------------------===//
// Ordered construct
//===----------------------------------------------------------------------===//
 
static LogicalResult verifyOrderedParent(Operation &op) {
  bool hasRegion = op.getNumRegions() > 0;
  auto loopOp = op.getParentOfType<LoopNestOp>();
  if (!loopOp) {
    if (hasRegion)
      return success();
 
    // TODO: Consider if this needs to be the case only for the standalone
    // variant of the ordered construct.
    return op.emitOpError() << "must be nested inside of a loop";
  }
 
  Operation *wrapper = loopOp->getParentOp();
  if (auto wsloopOp = dyn_cast<WsloopOp>(wrapper)) {
    IntegerAttr orderedAttr = wsloopOp.getOrderedAttr();
    if (!orderedAttr)
      return op.emitOpError() << "the enclosing worksharing-loop region must "
                                 "have an ordered clause";
 
    if (hasRegion && orderedAttr.getInt() != 0)
      return op.emitOpError() << "the enclosing loop's ordered clause must not "
                                 "have a parameter present";
 
    if (!hasRegion && orderedAttr.getInt() == 0)
      return op.emitOpError() << "the enclosing loop's ordered clause must "
                                 "have a parameter present";
  } else if (!isa<SimdOp>(wrapper)) {
    return op.emitOpError() << "must be nested inside of a worksharing, simd "
                               "or worksharing simd loop";
  }
  return success();
}
 
void OrderedOp::build(OpBuilder &builder, OperationState &state,
                      const OrderedOperands &clauses) {
  OrderedOp::build(builder, state, clauses.doacrossDependType,
                   clauses.doacrossNumLoops, clauses.doacrossDependVars);
}
 
LogicalResult OrderedOp::verify() {
  if (failed(verifyOrderedParent(**this)))
    return failure();
 
  auto wrapper = (*this)->getParentOfType<WsloopOp>();
  if (!wrapper || *wrapper.getOrdered() != *getDoacrossNumLoops())
    return emitOpError() << "number of variables in depend clause does not "
                         << "match number of iteration variables in the "
                         << "doacross loop";
 
  return success();
}
 
void OrderedRegionOp::build(OpBuilder &builder, OperationState &state,
                            const OrderedRegionOperands &clauses) {
  OrderedRegionOp::build(builder, state, clauses.parLevelSimd);
}
 
LogicalResult OrderedRegionOp::verify() { return verifyOrderedParent(**this); }
 
//===----------------------------------------------------------------------===//
// TaskwaitOp
//===----------------------------------------------------------------------===//
 
void TaskwaitOp::build(OpBuilder &builder, OperationState &state,
                       const TaskwaitOperands &clauses) {
  // TODO Store clauses in op: depend_iterated_kinds, depend_iterated, nowait.
  MLIRContext *ctx = builder.getContext();
  TaskwaitOp::build(
      builder, state,
      /*depend_kinds=*/makeArrayAttr(ctx, clauses.dependKinds),
      /*depend_vars=*/clauses.dependVars,
      /*depend_iterated_kinds=*/makeArrayAttr(ctx, clauses.dependIteratedKinds),
      /*depend_iterated=*/ValueRange(clauses.dependIterated),
      /*nowait=*/clauses.nowait);
}
 
//===----------------------------------------------------------------------===//
// Verifier for AtomicReadOp
//===----------------------------------------------------------------------===//
 
LogicalResult AtomicReadOp::verify() {
  if (verifyCommon().failed())
    return mlir::failure();
 
  int64_t version = 50;
  if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
    if (Attribute verAttr = moduleOp->getDiscardableAttr("omp.version"))
      version = llvm::cast<VersionAttr>(verAttr).getVersion();
 
  if (auto mo = getMemoryOrder()) {
    if (*mo == ClauseMemoryOrderKind::Release) {
      return emitError("memory-order must not be release for atomic reads");
    }
    if (*mo == ClauseMemoryOrderKind::Acq_rel) {
      // acq_rel is prohibited on read only in OpenMP 5.0; allowed in 5.1+.
      if (version < 51)
        return emitError("memory-order must not be acq_rel for atomic reads");
    }
  }
  return verifySynchronizationHint(*this, getHint());
}
 
//===----------------------------------------------------------------------===//
// Verifier for AtomicWriteOp
//===----------------------------------------------------------------------===//
 
LogicalResult AtomicWriteOp::verify() {
  if (verifyCommon().failed())
    return mlir::failure();
 
  int64_t version = 50;
  if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
    if (Attribute verAttr = moduleOp->getDiscardableAttr("omp.version"))
      version = llvm::cast<VersionAttr>(verAttr).getVersion();
 
  if (auto mo = getMemoryOrder()) {
    if (*mo == ClauseMemoryOrderKind::Acquire) {
      return emitError("memory-order must not be acquire for atomic writes");
    }
    if (*mo == ClauseMemoryOrderKind::Acq_rel) {
      // acq_rel is prohibited on write only in OpenMP 5.0; allowed in 5.1+.
      if (version < 51)
        return emitError("memory-order must not be acq_rel for atomic writes");
    }
  }
  return verifySynchronizationHint(*this, getHint());
}
 
//===----------------------------------------------------------------------===//
// Verifier for AtomicUpdateOp
//===----------------------------------------------------------------------===//
 
LogicalResult AtomicUpdateOp::canonicalize(AtomicUpdateOp op,
                                           PatternRewriter &rewriter) {
  if (op.isNoOp()) {
    rewriter.eraseOp(op);
    return success();
  }
  if (Value writeVal = op.getWriteOpVal()) {
    rewriter.replaceOpWithNewOp<AtomicWriteOp>(
        op, op.getX(), writeVal, op.getHintAttr(), op.getMemoryOrderAttr());
    return success();
  }
  return failure();
}
 
LogicalResult AtomicUpdateOp::verify() {
  if (verifyCommon().failed())
    return mlir::failure();
 
  int64_t version = 50;
  if (auto moduleOp = getOperation()->getParentOfType<ModuleOp>())
    if (Attribute verAttr = moduleOp->getDiscardableAttr("omp.version"))
      version = llvm::cast<VersionAttr>(verAttr).getVersion();
 
  if (auto mo = getMemoryOrder()) {
    if (*mo == ClauseMemoryOrderKind::Acq_rel ||
        *mo == ClauseMemoryOrderKind::Acquire) {
      // This restriction applies only to OpenMP 5.0; removed in 5.1.
      if (version < 51)
        return emitError(
            "memory-order must not be acq_rel or acquire for atomic updates");
    }
  }
 
  return verifySynchronizationHint(*this, getHint());
}
 
LogicalResult AtomicUpdateOp::verifyRegions() { return verifyRegionsCommon(); }
 
//===----------------------------------------------------------------------===//
// Verifier for AtomicCaptureOp
//===----------------------------------------------------------------------===//
 
AtomicReadOp AtomicCaptureOp::getAtomicReadOp() {
  if (auto op = dyn_cast<AtomicReadOp>(getFirstOp()))
    return op;
  return dyn_cast<AtomicReadOp>(getSecondOp());
}
 
AtomicWriteOp AtomicCaptureOp::getAtomicWriteOp() {
  if (auto op = dyn_cast<AtomicWriteOp>(getFirstOp()))
    return op;
  return dyn_cast<AtomicWriteOp>(getSecondOp());
}
 
AtomicUpdateOp AtomicCaptureOp::getAtomicUpdateOp() {
  if (auto op = dyn_cast<AtomicUpdateOp>(getFirstOp()))
    return op;
  return dyn_cast<AtomicUpdateOp>(getSecondOp());
}
 
AtomicCompareOp AtomicCaptureOp::getAtomicCompareOp() {
  if (auto op = dyn_cast<AtomicCompareOp>(getFirstOp()))
    return op;
  return dyn_cast<AtomicCompareOp>(getSecondOp());
}
 
LogicalResult AtomicCaptureOp::verify() {
  return verifySynchronizationHint(*this, getHint());
}
 
LogicalResult AtomicCaptureOp::verifyRegions() {
  if (verifyRegionsCommon().failed())
    return mlir::failure();
 
  if (getFirstOp()->getInherentAttr("hint").value_or(Attribute{}) ||
      getSecondOp()->getInherentAttr("hint").value_or(Attribute{}))
    return emitOpError(
        "operations inside capture region must not have hint clause");
 
  if (getFirstOp()->getInherentAttr("memory_order").value_or(Attribute{}) ||
      getSecondOp()->getInherentAttr("memory_order").value_or(Attribute{}))
    return emitOpError(
        "operations inside capture region must not have memory_order clause");
  return success();
}
 
//===----------------------------------------------------------------------===//
// AtomicCompareOp
//===----------------------------------------------------------------------===//
 
LogicalResult AtomicCompareOp::verify() {
  if (verifyCommon().failed())
    return mlir::failure();
  // OpenMP 5.2 [15.8.3]: the fail clause argument must be one of seq_cst,
  // acquire or relaxed ('release' and 'acq_rel' are not valid failure
  // orderings and map to invalid cmpxchg failure orderings).
  if (auto failOrder = getFailMemoryOrder()) {
    if (*failOrder != ClauseMemoryOrderKind::Seq_cst &&
        *failOrder != ClauseMemoryOrderKind::Acquire &&
        *failOrder != ClauseMemoryOrderKind::Relaxed)
      return emitOpError(
          "fail_memory_order must be 'seq_cst', 'acquire' or 'relaxed'");
  }
  return verifySynchronizationHint(*this, getHint());
}
 
LogicalResult AtomicCompareOp::verifyRegions() {
  if (verifyRegionsCommon().failed())
    return mlir::failure();
 
  if (verifyOperator().failed())
    return mlir::failure();
 
  Block &block = getRegion().front();
 
  Operation *terminator = block.getTerminator();
  if (!terminator || !isa<YieldOp>(terminator))
    return emitOpError("region must be terminated with omp.yield");
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// CancelOp
//===----------------------------------------------------------------------===//
 
void CancelOp::build(OpBuilder &builder, OperationState &state,
                     const CancelOperands &clauses) {
  CancelOp::build(builder, state, clauses.cancelDirective, clauses.ifExpr);
}
 
static Operation *getParentInSameDialect(Operation *thisOp) {
  Operation *parent = thisOp->getParentOp();
  while (parent) {
    if (parent->getDialect() == thisOp->getDialect())
      return parent;
    parent = parent->getParentOp();
  }
  return nullptr;
}
 
LogicalResult CancelOp::verify() {
  ClauseCancellationConstructType cct = getCancelDirective();
  // The next OpenMP operation in the chain of parents
  Operation *structuralParent = getParentInSameDialect((*this).getOperation());
  if (!structuralParent)
    return emitOpError() << "Orphaned cancel construct";
 
  if ((cct == ClauseCancellationConstructType::Parallel) &&
      !mlir::isa<ParallelOp>(structuralParent)) {
    return emitOpError() << "cancel parallel must appear "
                         << "inside a parallel region";
  }
  if (cct == ClauseCancellationConstructType::Loop) {
    // structural parent will be omp.loop_nest, directly nested inside
    // omp.wsloop
    auto wsloopOp = mlir::dyn_cast<WsloopOp>(structuralParent->getParentOp());
 
    if (!wsloopOp) {
      return emitOpError()
             << "cancel loop must appear inside a worksharing-loop region";
    }
    if (wsloopOp.getNowaitAttr()) {
      return emitError() << "A worksharing construct that is canceled "
                         << "must not have a nowait clause";
    }
    if (wsloopOp.getOrderedAttr()) {
      return emitError() << "A worksharing construct that is canceled "
                         << "must not have an ordered clause";
    }
 
  } else if (cct == ClauseCancellationConstructType::Sections) {
    // structural parent will be an omp.section, directly nested inside
    // omp.sections
    auto sectionsOp =
        mlir::dyn_cast<SectionsOp>(structuralParent->getParentOp());
    if (!sectionsOp) {
      return emitOpError() << "cancel sections must appear "
                           << "inside a sections region";
    }
    if (sectionsOp.getNowait()) {
      return emitError() << "A sections construct that is canceled "
                         << "must not have a nowait clause";
    }
  }
  if ((cct == ClauseCancellationConstructType::Taskgroup) &&
      (!mlir::isa<omp::TaskOp>(structuralParent) &&
       !mlir::isa<omp::TaskloopWrapperOp>(structuralParent->getParentOp()))) {
    return emitOpError() << "cancel taskgroup must appear "
                         << "inside a task region";
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// CancellationPointOp
//===----------------------------------------------------------------------===//
 
void CancellationPointOp::build(OpBuilder &builder, OperationState &state,
                                const CancellationPointOperands &clauses) {
  CancellationPointOp::build(builder, state, clauses.cancelDirective);
}
 
LogicalResult CancellationPointOp::verify() {
  ClauseCancellationConstructType cct = getCancelDirective();
  // The next OpenMP operation in the chain of parents
  Operation *structuralParent = getParentInSameDialect((*this).getOperation());
  if (!structuralParent)
    return emitOpError() << "Orphaned cancellation point";
 
  if ((cct == ClauseCancellationConstructType::Parallel) &&
      !mlir::isa<ParallelOp>(structuralParent)) {
    return emitOpError() << "cancellation point parallel must appear "
                         << "inside a parallel region";
  }
  // Strucutal parent here will be an omp.loop_nest. Get the parent of that to
  // find the wsloop
  if ((cct == ClauseCancellationConstructType::Loop) &&
      !mlir::isa<WsloopOp>(structuralParent->getParentOp())) {
    return emitOpError() << "cancellation point loop must appear "
                         << "inside a worksharing-loop region";
  }
  if ((cct == ClauseCancellationConstructType::Sections) &&
      !mlir::isa<omp::SectionOp>(structuralParent)) {
    return emitOpError() << "cancellation point sections must appear "
                         << "inside a sections region";
  }
  if ((cct == ClauseCancellationConstructType::Taskgroup) &&
      (!mlir::isa<omp::TaskOp>(structuralParent) &&
       !mlir::isa<omp::TaskloopWrapperOp>(structuralParent->getParentOp()))) {
    return emitOpError() << "cancellation point taskgroup must appear "
                         << "inside a task region";
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// MapBoundsOp
//===----------------------------------------------------------------------===//
 
LogicalResult MapBoundsOp::verify() {
  auto extent = getExtent();
  auto upperbound = getUpperBound();
  if (!extent && !upperbound)
    return emitError("expected extent or upperbound.");
  return success();
}
 
void PrivateClauseOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                            TypeRange /*result_types*/, StringAttr symName,
                            TypeAttr type) {
  PrivateClauseOp::build(
      odsBuilder, odsState, symName, /*sym_visibility=*/nullptr, type,
      DataSharingClauseTypeAttr::get(odsBuilder.getContext(),
                                     DataSharingClauseType::Private));
}
 
LogicalResult PrivateClauseOp::verifyRegions() {
  Type argType = getArgType();
  auto verifyTerminator = [&](Operation *terminator, bool yieldsValue,
                              bool yieldsAnyType) -> LogicalResult {
    if (!terminator->getBlock()->getSuccessors().empty())
      return success();
 
    if (!llvm::isa<YieldOp>(terminator))
      return mlir::emitError(terminator->getLoc())
             << "expected exit block terminator to be an `omp.yield` op.";
 
    YieldOp yieldOp = llvm::cast<YieldOp>(terminator);
    TypeRange yieldedTypes = yieldOp.getResults().getTypes();
 
    if (!yieldsValue) {
      if (yieldedTypes.empty())
        return success();
 
      return mlir::emitError(terminator->getLoc())
             << "Did not expect any values to be yielded.";
    }
 
    if (yieldedTypes.size() == 1 &&
        (yieldsAnyType || yieldedTypes.front() == argType))
      return success();
 
    auto error = mlir::emitError(yieldOp.getLoc())
                 << "Invalid yielded value. Expected ";
    if (yieldsAnyType)
      error << "a single value";
    else
      error << "type: " << argType;
    error << ", got: ";
 
    if (yieldedTypes.empty())
      error << "None";
    else
      error << yieldedTypes;
 
    return error;
  };
 
  auto verifyRegion = [&](Region &region, unsigned expectedNumArgs,
                          StringRef regionName, bool yieldsValue,
                          bool yieldsAnyType = false) -> LogicalResult {
    assert(!region.empty());
 
    if (region.getNumArguments() != expectedNumArgs)
      return mlir::emitError(region.getLoc())
             << "`" << regionName << "`: " << "expected " << expectedNumArgs
             << " region arguments, got: " << region.getNumArguments();
 
    for (Block &block : region) {
      // MLIR will verify the absence of the terminator for us.
      if (!block.mightHaveTerminator())
        continue;
 
      if (failed(verifyTerminator(block.getTerminator(), yieldsValue,
                                  yieldsAnyType)))
        return failure();
    }
 
    return success();
  };
 
  // Ensure all of the region arguments have the same type
  for (Region *region : getRegions())
    for (Type ty : region->getArgumentTypes())
      if (ty != argType)
        return emitError() << "Region argument type mismatch: got " << ty
                           << " expected " << argType << ".";
 
  mlir::Region &initRegion = getInitRegion();
  if (!initRegion.empty() &&
      failed(verifyRegion(getInitRegion(), /*expectedNumArgs=*/2, "init",
                          /*yieldsValue=*/true)))
    return failure();
 
  DataSharingClauseType dsType = getDataSharingType();
 
  if (dsType == DataSharingClauseType::Private && !getCopyRegion().empty())
    return emitError("`private` clauses do not require a `copy` region.");
 
  if (dsType == DataSharingClauseType::FirstPrivate && getCopyRegion().empty())
    return emitError(
        "`firstprivate` clauses require at least a `copy` region.");
 
  if (dsType == DataSharingClauseType::FirstPrivate &&
      failed(verifyRegion(getCopyRegion(), /*expectedNumArgs=*/2, "copy",
                          /*yieldsValue=*/true)))
    return failure();
 
  if (dsType == DataSharingClauseType::Linear && !getCopyRegion().empty())
    return emitError("`linear` clauses do not require a `copy` region.");
 
  if (!getDataAddressRegion().empty()) {
    if (dsType != DataSharingClauseType::Linear)
      return emitError(
          "only `linear` clauses can have a `data_address` region.");
    if (failed(verifyRegion(getDataAddressRegion(), /*expectedNumArgs=*/1,
                            "data_address", /*yieldsValue=*/true,
                            /*yieldsAnyType=*/true)))
      return failure();
  }
 
  if (!getDeallocRegion().empty() &&
      failed(verifyRegion(getDeallocRegion(), /*expectedNumArgs=*/1, "dealloc",
                          /*yieldsValue=*/false)))
    return failure();
 
  return success();
}
 
//===----------------------------------------------------------------------===//
// Spec 5.2: Masked construct (10.5)
//===----------------------------------------------------------------------===//
 
void MaskedOp::build(OpBuilder &builder, OperationState &state,
                     const MaskedOperands &clauses) {
  MaskedOp::build(builder, state, clauses.filteredThreadId);
}
 
//===----------------------------------------------------------------------===//
// Spec 5.2: Dispatch construct (7.6)
//===----------------------------------------------------------------------===//
 
void DispatchOp::build(OpBuilder &builder, OperationState &state,
                       const DispatchOperands &clauses) {
  DispatchOp::build(builder, state, clauses.nocontext, clauses.novariants,
                    clauses.nowait);
}
 
//===----------------------------------------------------------------------===//
// Spec 5.2: Scan construct (5.6)
//===----------------------------------------------------------------------===//
 
void ScanOp::build(OpBuilder &builder, OperationState &state,
                   const ScanOperands &clauses) {
  ScanOp::build(builder, state, clauses.inclusiveVars, clauses.exclusiveVars);
}
 
LogicalResult ScanOp::verify() {
  if (hasExclusiveVars() == hasInclusiveVars())
    return emitError(
        "Exactly one of EXCLUSIVE or INCLUSIVE clause is expected");
  if (WsloopOp parentWsLoopOp = (*this)->getParentOfType<WsloopOp>()) {
    if (parentWsLoopOp.getReductionModAttr() &&
        parentWsLoopOp.getReductionModAttr().getValue() ==
            ReductionModifier::inscan)
      return success();
  }
  if (SimdOp parentSimdOp = (*this)->getParentOfType<SimdOp>()) {
    if (parentSimdOp.getReductionModAttr() &&
        parentSimdOp.getReductionModAttr().getValue() ==
            ReductionModifier::inscan)
      return success();
  }
  return emitError("SCAN directive needs to be enclosed within a parent "
                   "worksharing loop construct or SIMD construct with INSCAN "
                   "reduction modifier");
}
 
/// Verifies align clause in allocate directive
LogicalResult verifyAlignment(Operation &op,
                              std::optional<uint64_t> alignment) {
  if (alignment.has_value()) {
    if ((alignment.value() != 0) && !llvm::has_single_bit(alignment.value()))
      return op.emitError()
             << "ALIGN value : " << alignment.value() << " must be power of 2";
  }
  return success();
}
 
LogicalResult AllocateDirOp::verify() {
  return verifyAlignment(*getOperation(), getAlign());
}
 
//===----------------------------------------------------------------------===//
// AllocSharedMemOp
//===----------------------------------------------------------------------===//
 
LogicalResult AllocSharedMemOp::verify() {
  return verifyAlignment(*getOperation(), getMemAlignment());
}
 
//===----------------------------------------------------------------------===//
// FreeSharedMemOp
//===----------------------------------------------------------------------===//
 
LogicalResult FreeSharedMemOp::verify() {
  return verifyAlignment(*getOperation(), getMemAlignment());
}
 
//===----------------------------------------------------------------------===//
// WorkdistributeOp
//===----------------------------------------------------------------------===//
 
LogicalResult WorkdistributeOp::verify() {
  if (isCombined())
    return emitOpError() << "cannot be a non-innermost combined construct leaf";
 
  // Check that region exists and is not empty
  Region &region = getRegion();
  if (region.empty())
    return emitOpError("region cannot be empty");
  // Verify single entry point.
  Block &entryBlock = region.front();
  if (entryBlock.empty())
    return emitOpError("region must contain a structured block");
  // Verify single exit point.
  bool hasTerminator = false;
  for (Block &block : region) {
    if (isa<TerminatorOp>(block.back())) {
      if (hasTerminator) {
        return emitOpError("region must have exactly one terminator");
      }
      hasTerminator = true;
    }
  }
  if (!hasTerminator) {
    return emitOpError("region must be terminated with omp.terminator");
  }
  auto walkResult = region.walk([&](Operation *op) -> WalkResult {
    // No implicit barrier at end
    if (isa<BarrierOp>(op)) {
      return emitOpError(
          "explicit barriers are not allowed in workdistribute region");
    }
    // Check for invalid nested constructs
    if (isa<ParallelOp>(op)) {
      return emitOpError(
          "nested parallel constructs not allowed in workdistribute");
    }
    if (isa<TeamsOp>(op)) {
      return emitOpError(
          "nested teams constructs not allowed in workdistribute");
    }
    return WalkResult::advance();
  });
  if (walkResult.wasInterrupted())
    return failure();
 
  Operation *parentOp = (*this)->getParentOp();
  if (!llvm::dyn_cast<TeamsOp>(parentOp))
    return emitOpError("workdistribute must be nested under teams");
  return success();
}
 
//===----------------------------------------------------------------------===//
// Declare simd [7.7]
//===----------------------------------------------------------------------===//
 
LogicalResult DeclareSimdOp::verify() {
  // Must be nested inside a function-like op
  auto func =
      dyn_cast_if_present<mlir::FunctionOpInterface>((*this)->getParentOp());
  if (!func)
    return emitOpError() << "must be nested inside a function";
 
  if (getInbranch() && getNotinbranch())
    return emitOpError("cannot have both 'inbranch' and 'notinbranch'");
 
  if (failed(verifyLinearModifiers(*this, getLinearModifiers(), getLinearVars(),
                                   /*isDeclareSimd=*/true)))
    return failure();
 
  if (getLinearSyms())
    return emitOpError("linear privatizers are not allowed");
 
  return verifyAlignedClause(*this, getAlignments(), getAlignedVars());
}
 
void DeclareSimdOp::build(OpBuilder &odsBuilder, OperationState &odsState,
                          const DeclareSimdOperands &clauses) {
  MLIRContext *ctx = odsBuilder.getContext();
  DeclareSimdOp::build(
      odsBuilder, odsState, clauses.alignedVars,
      makeArrayAttr(ctx, clauses.alignments), clauses.inbranch,
      clauses.linearVars, clauses.linearStepVars, clauses.linearVarTypes,
      clauses.linearModifiers, makeArrayAttr(ctx, clauses.linearSyms),
      clauses.notinbranch, clauses.simdlen, clauses.uniformVars);
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for Uniform Clause
//===----------------------------------------------------------------------===//
 
/// uniform ::= `uniform` `(` uniform-list `)`
/// uniform-list := uniform-val (`,` uniform-val)*
/// uniform-val := ssa-id `:` type
static ParseResult
parseUniformClause(OpAsmParser &parser,
                   SmallVectorImpl<OpAsmParser::UnresolvedOperand> &uniformVars,
                   SmallVectorImpl<Type> &uniformTypes) {
  return parser.parseCommaSeparatedList([&]() -> mlir::ParseResult {
    if (parser.parseOperand(uniformVars.emplace_back()) ||
        parser.parseColonType(uniformTypes.emplace_back()))
      return mlir::failure();
    return mlir::success();
  });
}
 
/// Print Uniform Clauses
static void printUniformClause(OpAsmPrinter &p, Operation *op,
                               ValueRange uniformVars, TypeRange uniformTypes) {
  for (unsigned i = 0; i < uniformVars.size(); ++i) {
    if (i != 0)
      p << ", ";
    p << uniformVars[i] << " : " << uniformTypes[i];
  }
}
 
//===----------------------------------------------------------------------===//
// Parser and printer for Affinity Clause
//===----------------------------------------------------------------------===//
 
static ParseResult parseAffinityClause(
    OpAsmParser &parser,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &iterated,
    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &affinityVars,
    SmallVectorImpl<Type> &iteratedTypes,
    SmallVectorImpl<Type> &affinityVarTypes) {
  if (failed(parseSplitIteratedList(
          parser, iterated, iteratedTypes, affinityVars, affinityVarTypes,
          /*parsePrefix=*/[&]() -> ParseResult { return success(); })))
    return failure();
  return success();
}
 
static void printAffinityClause(OpAsmPrinter &p, Operation *op,
                                ValueRange iterated, ValueRange affinityVars,
                                TypeRange iteratedTypes,
                                TypeRange affinityVarTypes) {
  auto nop = [&](Value, Type) {};
  printSplitIteratedList(p, iterated, iteratedTypes, affinityVars,
                         affinityVarTypes,
                         /*plain prefix*/ nop,
                         /*iterated prefix*/ nop);
}
 
//===----------------------------------------------------------------------===//
// Parser, printer, and verifier for Iterator modifier
//===----------------------------------------------------------------------===//
 
static ParseResult
parseIteratorHeader(OpAsmParser &parser, Region &region,
                    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &lbs,
                    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &ubs,
                    SmallVectorImpl<OpAsmParser::UnresolvedOperand> &steps,
                    SmallVectorImpl<Type> &lbTypes,
                    SmallVectorImpl<Type> &ubTypes,
                    SmallVectorImpl<Type> &stepTypes) {
 
  llvm::SMLoc ivLoc = parser.getCurrentLocation();
  SmallVector<OpAsmParser::Argument> ivArgs;
 
  // Parse induction variables: %i : i32, %j : i32
  if (parser.parseCommaSeparatedList([&]() -> ParseResult {
        OpAsmParser::Argument &arg = ivArgs.emplace_back();
        if (parser.parseArgument(arg))
          return failure();
 
        // Optional type, default to Index if not provided
        if (succeeded(parser.parseOptionalColon())) {
          if (parser.parseType(arg.type))
            return failure();
        } else {
          arg.type = parser.getBuilder().getIndexType();
        }
        return success();
      }))
    return failure();
 
  // ) = (
  if (parser.parseRParen() || parser.parseEqual() || parser.parseLParen())
    return failure();
 
  // Parse Ranges: (%lb to %ub step %st, ...)
  if (parser.parseCommaSeparatedList([&]() -> ParseResult {
        OpAsmParser::UnresolvedOperand lb, ub, st;
        if (parser.parseOperand(lb) || parser.parseKeyword("to") ||
            parser.parseOperand(ub) || parser.parseKeyword("step") ||
            parser.parseOperand(st))
          return failure();
 
        lbs.push_back(lb);
        ubs.push_back(ub);
        steps.push_back(st);
        return success();
      }))
    return failure();
 
  if (parser.parseRParen())
    return failure();
 
  if (ivArgs.size() != lbs.size())
    return parser.emitError(ivLoc)
           << "mismatch: " << ivArgs.size() << " variables but " << lbs.size()
           << " ranges";
 
  for (auto &arg : ivArgs) {
    lbTypes.push_back(arg.type);
    ubTypes.push_back(arg.type);
    stepTypes.push_back(arg.type);
  }
 
  return parser.parseRegion(region, ivArgs);
}
 
static void printIteratorHeader(OpAsmPrinter &p, Operation *op, Region &region,
                                ValueRange lbs, ValueRange ubs,
                                ValueRange steps, TypeRange, TypeRange,
                                TypeRange) {
  Block &entry = region.front();
 
  for (unsigned i = 0, e = entry.getNumArguments(); i < e; ++i) {
    if (i != 0)
      p << ", ";
    p.printRegionArgument(entry.getArgument(i));
  }
  p << ") = (";
 
  // (%lb0 to %ub0 step %step0, %lb1 to %ub1 step %step1, ...)
  for (unsigned i = 0, e = lbs.size(); i < e; ++i) {
    if (i)
      p << ", ";
    p << lbs[i] << " to " << ubs[i] << " step " << steps[i];
  }
  p << ") ";
 
  p.printRegion(region, /*printEntryBlockArgs=*/false,
                /*printBlockTerminators=*/true);
}
 
YieldOp IteratorOp::getYieldOp() {
  YieldOp yield;
  for (Block &block : getRegion()) {
    auto blockYield =
        dyn_cast_if_present<YieldOp>(block.empty() ? nullptr : &block.back());
    if (!blockYield)
      continue;
    if (yield)
      return {};
    yield = blockYield;
  }
  return yield;
}
 
LogicalResult IteratorOp::verify() {
  auto iteratedTy = llvm::dyn_cast<omp::IteratedType>(getIterated().getType());
  if (!iteratedTy)
    return emitOpError() << "result must be omp.iterated<entry_ty>";
 
  Block::BlockArgListType ivs = getRegion().front().getArguments();
  if (getLoopLowerBounds().size() != ivs.size())
    return emitOpError() << "number of range arguments and IVs do not match";
 
  for (auto [lb, iv] : llvm::zip_equal(getLoopLowerBounds(), ivs)) {
    if (lb.getType() != iv.getType())
      return emitOpError()
             << "range argument type does not match corresponding IV type";
  }
 
  for (Value step : getLoopSteps()) {
    if (matchPattern(step, m_Zero()))
      return emitOpError() << "loop step must not be zero";
  }
 
  YieldOp yield = getYieldOp();
  if (!yield) {
    bool hasYield = llvm::any_of(getRegion(), [](Block &block) {
      return !block.empty() && isa<YieldOp>(block.back());
    });
    if (hasYield)
      return emitOpError() << "region must contain exactly one omp.yield";
    return emitOpError() << "region must be terminated by omp.yield";
  }
 
  if (yield.getNumOperands() != 1)
    return emitOpError()
           << "omp.yield in omp.iterator region must yield exactly one value";
 
  mlir::Type yieldedTy = yield.getOperand(0).getType();
  mlir::Type elemTy = iteratedTy.getElementType();
 
  if (yieldedTy != elemTy)
    return emitOpError() << "omp.iterated element type (" << elemTy
                         << ") does not match omp.yield operand type ("
                         << yieldedTy << ")";
 
  return success();
}
 
LogicalResult IteratorOp::verifyRegions() {
  // Translation routes only the omp.yield block back to the iterator loop.
  for (Block &block : getRegion()) {
    Operation *terminator = block.getTerminator();
    if (block.getNumSuccessors() == 0 && !isa<YieldOp>(terminator))
      return mlir::emitError(terminator->getLoc())
             << "expected exit block terminator to be an `omp.yield` op.";
  }
  return success();
}
 
//===----------------------------------------------------------------------===//
// GroupprivateOp
//===----------------------------------------------------------------------===//
 
LogicalResult
GroupprivateOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
  auto *symbol = symbolTable.lookupNearestSymbolFrom(*this, getSymNameAttr());
  if (!symbol)
    return emitOpError() << "expected symbol reference '" << getSymName()
                         << "' to point to a global variable";
 
  if (isa<FunctionOpInterface>(symbol))
    return emitOpError() << "expected symbol reference '" << getSymName()
                         << "' to point to a global variable, not a function";
 
  return success();
}
 
#define GET_ATTRDEF_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOpsAttributes.cpp.inc"
 
#define GET_OP_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOps.cpp.inc"
 
#define GET_TYPEDEF_CLASSES
#include "mlir/Dialect/OpenMP/OpenMPOpsTypes.cpp.inc"