28#include "llvm/ADT/DenseMap.h"
29#include "llvm/ADT/STLExtras.h"
30#include "llvm/ADT/SmallVector.h"
56struct RemoveEmptyKernelEnvironment
58 using OpRewritePattern<acc::KernelEnvironmentOp>::OpRewritePattern;
60 LogicalResult matchAndRewrite(acc::KernelEnvironmentOp op,
61 PatternRewriter &rewriter)
const override {
62 assert(op->getNumRegions() == 1 &&
"expected op to have one region");
64 Block &block = op.getRegion().front();
70 if (!op.getWaitOperands().empty() || op.getWaitOnlyAttr())
72 op, op.getWaitOperands(), Value(),
73 op.getWaitDevnum(),
nullptr, Value());
81struct KernelEnvironmentDedupDataOperands
83 using OpRewritePattern<acc::KernelEnvironmentOp>::OpRewritePattern;
85 static bool isReductionDataOperand(Value operand) {
90 if (!clause || *clause != DataClause::acc_reduction)
92 assert(isa<CopyinOp>(def) &&
"reduction clause must be an acc.copyin");
96 LogicalResult matchAndRewrite(acc::KernelEnvironmentOp op,
97 PatternRewriter &rewriter)
const override {
98 SmallVector<Value> operands(op.getDataClauseOperands());
99 const unsigned numOperands = operands.size();
103 SmallVector<Value> kept;
104 SmallVector<std::pair<Value, Value>> useRewrites;
105 SmallVector<SmallVector<Value>> boundsStorage;
106 boundsStorage.reserve(numOperands);
107 llvm::DenseMap<std::pair<Value, ArrayRef<Value>>,
unsigned>
108 keptIndexByVarAndBounds;
109 for (Value operand : operands) {
114 if (!def || isa<MapInfoOp>(def)) {
115 kept.push_back(operand);
121 kept.push_back(operand);
126 auto [it,
inserted] = keptIndexByVarAndBounds.try_emplace(
127 std::make_pair(var, ArrayRef<Value>(boundsStorage.back())),
130 kept.push_back(operand);
134 Value &survivor = kept[it->second];
137 if (isReductionDataOperand(operand) && !isReductionDataOperand(survivor))
138 std::swap(survivor, operand);
139 useRewrites.emplace_back(operand, survivor);
141 if (kept.size() == numOperands)
144 for (
auto [from, to] : useRewrites) {
146 return op->isProperAncestor(use.
getOwner());
150 op, [&] { op.getDataClauseOperandsMutable().assign(kept); });
157static void setComputeRegionInputOperandSegments(ComputeRegionOp op,
161 const size_t numLaunch = op.getLaunchArgs().size();
163 rewriter.
getStringAttr(ComputeRegionOp::getOperandSegmentSizeAttr()),
165 static_cast<int32_t>(numInput),
166 hasStream ? 1 : 0}));
169struct ComputeRegionRemoveDuplicateArgs
173 LogicalResult matchAndRewrite(ComputeRegionOp op,
174 PatternRewriter &rewriter)
const override {
175 Block *body = op.getBody();
176 const size_t numLaunch = op.getLaunchArgs().size();
177 size_t numInput = op.getInputArgs().size();
179 "region args mismatch");
181 bool hasDuplicate =
false;
182 for (
size_t j = 1; j < numInput && !hasDuplicate; ++j)
183 for (
size_t i = 0; i < j; ++i)
184 if (op->getOperand(
static_cast<unsigned>(numLaunch + i)) ==
185 op->getOperand(
static_cast<unsigned>(numLaunch + j))) {
192 const bool hasStream =
static_cast<bool>(op.getStream());
196 for (
size_t j = 1; j < numInput && !merged; ++j) {
197 for (
size_t i = 0; i < j; ++i) {
198 if (op->getOperand(
static_cast<unsigned>(numLaunch + i)) !=
199 op->getOperand(
static_cast<unsigned>(numLaunch + j)))
201 unsigned keepIdx =
static_cast<unsigned>(numLaunch + i);
202 unsigned dropIdx =
static_cast<unsigned>(numLaunch + j);
206 op->eraseOperand(dropIdx);
215 setComputeRegionInputOperandSegments(op, rewriter, numInput, hasStream);
221struct ComputeRegionRemoveUnusedArgs
225 LogicalResult matchAndRewrite(ComputeRegionOp op,
226 PatternRewriter &rewriter)
const override {
227 Block *body = op.getBody();
228 const size_t numLaunch = op.getLaunchArgs().size();
229 size_t numInput = op.getInputArgs().size();
231 "region args mismatch");
233 bool hasUnused =
false;
234 for (
size_t k = numLaunch; k < numLaunch + numInput; ++k)
242 const bool hasStream =
static_cast<bool>(op.getStream());
244 for (
size_t k = numLaunch; k < numLaunch + numInput;) {
250 op->eraseOperand(
static_cast<unsigned>(k));
253 setComputeRegionInputOperandSegments(op, rewriter, numInput, hasStream);
259template <
typename EffectTy>
264 for (
unsigned i = 0, e = operand.
size(); i < e; ++i)
265 effects.emplace_back(EffectTy::get(), &operand[i]);
268template <
typename EffectTy>
273 effects.emplace_back(EffectTy::get(), mlir::cast<mlir::OpResult>(
result));
276static int64_t gpuProcessorIndex(gpu::Processor p) {
278 case gpu::Processor::Sequential:
280 case gpu::Processor::ThreadX:
282 case gpu::Processor::ThreadY:
284 case gpu::Processor::ThreadZ:
286 case gpu::Processor::BlockX:
288 case gpu::Processor::BlockY:
290 case gpu::Processor::BlockZ:
293 llvm_unreachable(
"unhandled gpu::Processor");
296static gpu::Processor indexToGpuProcessor(
int64_t idx) {
299 return gpu::Processor::Sequential;
301 return gpu::Processor::ThreadX;
303 return gpu::Processor::ThreadY;
305 return gpu::Processor::ThreadZ;
307 return gpu::Processor::BlockX;
309 return gpu::Processor::BlockY;
311 return gpu::Processor::BlockZ;
313 return gpu::Processor::Sequential;
318 return GPUParallelDimAttr::get(
319 context, IntegerAttr::get(IndexType::get(context), dimInt));
322static GPUParallelDimAttr processorParDim(
MLIRContext *context,
323 gpu::Processor proc) {
324 return GPUParallelDimAttr::get(
326 IntegerAttr::get(IndexType::get(context), gpuProcessorIndex(proc)));
329static ParseResult parseProcessorValue(
AsmParser &parser,
330 GPUParallelDimAttr &dim) {
335 auto maybeProcessor = gpu::symbolizeProcessor(keyword);
338 <<
"expected one of ::mlir::gpu::Processor enum names";
339 dim = intToParDim(parser.
getContext(), gpuProcessorIndex(*maybeProcessor));
343static void printProcessorValue(
AsmPrinter &printer,
344 const GPUParallelDimAttr &attr) {
345 gpu::Processor processor = indexToGpuProcessor(attr.getValue().getInt());
346 printer << gpu::stringifyProcessor(processor);
349static FailureOr<SmallVector<GPUParallelDimAttr>>
350parseGPUParallelDimList(
AsmParser &parser) {
352 auto parseParDim = [&]() -> ParseResult {
353 GPUParallelDimAttr dim;
354 if (parseProcessorValue(parser, dim))
356 parDims.push_back(dim);
360 "list of OpenACC GPU parallel dimensions"))
365static void printGPUParallelDimList(
AsmPrinter &printer,
368 llvm::interleaveComma(dims, printer, [&printer](
const GPUParallelDimAttr &p) {
369 printProcessorValue(printer, p);
379void KernelEnvironmentOp::getSuccessorRegions(
389void KernelEnvironmentOp::getCanonicalizationPatterns(
391 results.
add<RemoveEmptyKernelEnvironment, KernelEnvironmentDedupDataOperands>(
396template <
typename ComputeConstructT>
400 std::optional<Value> &asyncOperand, UnitAttr &asyncOnly) {
401 if (computeConstruct.hasAsyncOnly(clauseDeviceType)) {
402 asyncOnly = UnitAttr::get(context);
405 if (
Value asyncValue = computeConstruct.getAsyncValue(clauseDeviceType)) {
406 asyncOperand = asyncValue;
413template <
typename ComputeConstructT>
416 std::optional<Value> &waitDevnum,
418 UnitAttr &waitOnly) {
419 if (computeConstruct.hasWaitOnly(clauseDeviceType)) {
420 waitOnly = UnitAttr::get(context);
423 Value devnum = computeConstruct.getWaitDevnum(clauseDeviceType);
424 auto waitValues = computeConstruct.getWaitValues(clauseDeviceType);
425 if (!devnum && waitValues.empty())
429 waitOperands.append(waitValues.begin(), waitValues.end());
433template <
typename ComputeConstructT>
435 ComputeConstructT computeConstruct, DeviceType deviceType,
436 std::optional<Value> &asyncOperand, UnitAttr &asyncOnly,
438 UnitAttr &waitOnly) {
439 MLIRContext *context = computeConstruct->getContext();
444 if (deviceType != DeviceType::None)
446 asyncOperand, asyncOnly);
450 waitOperands, waitOnly)) {
451 if (deviceType != DeviceType::None)
453 waitOperands, waitOnly);
457template <
typename ComputeConstructT>
459KernelEnvironmentOp::createAndPopulate(ComputeConstructT computeConstruct,
460 DeviceType deviceType,
462 std::optional<Value> asyncOperand;
463 UnitAttr asyncOnly =
nullptr;
464 std::optional<Value> waitDevnum;
466 UnitAttr waitOnly =
nullptr;
468 asyncOnly, waitDevnum, waitOperands,
471 auto kernelEnvironment = KernelEnvironmentOp::create(
472 builder, computeConstruct->getLoc(),
473 computeConstruct.getDataClauseOperands(), asyncOperand.value_or(
Value()),
474 asyncOnly, waitDevnum.value_or(
Value()), waitOperands, waitOnly);
475 Block &block = kernelEnvironment.getRegion().emplaceBlock();
477 return kernelEnvironment;
480template KernelEnvironmentOp
481KernelEnvironmentOp::createAndPopulate<ParallelOp>(ParallelOp, DeviceType,
483template KernelEnvironmentOp
484KernelEnvironmentOp::createAndPopulate<KernelsOp>(KernelsOp, DeviceType,
486template KernelEnvironmentOp
487KernelEnvironmentOp::createAndPopulate<SerialOp>(SerialOp, DeviceType,
490LogicalResult KernelEnvironmentOp::verify() {
492 return emitError(
"async-only cannot appear with async operand");
493 if (getWaitOnly() && (!getWaitOperands().empty() || getWaitDevnum()))
494 return emitError(
"wait-only cannot appear with wait operands or devnum");
495 if ((*this)->getParentOfType<KernelEnvironmentOp>())
497 "must not be nested within another acc.kernel_environment");
505LogicalResult FirstprivateMapInitialOp::verify() {
507 return emitError(
"data clause associated with firstprivate operation must "
510 return emitError(
"must have var operand");
511 if (!mlir::isa<mlir::acc::PointerLikeType>(
getVar().
getType()) &&
513 return emitError(
"var must be mappable or pointer-like");
514 if (mlir::isa<mlir::acc::PointerLikeType>(
getVar().
getType()) &&
516 return emitError(
"varType must capture the element type of var");
517 if (getModifiers() != acc::DataClauseModifier::none)
518 return emitError(
"no data clause modifiers are allowed");
522void FirstprivateMapInitialOp::getEffects(
535void ReductionInitOp::getSuccessorRegions(
541void ReductionInitOp::getRegionInvocationBounds(
544 invocationBounds.emplace_back(1, 1);
551LogicalResult ReductionInitOp::verify() {
553 if (
auto yieldOp = dyn_cast<acc::YieldOp>(block.
getTerminator())) {
554 if (yieldOp.getNumOperands() != 1)
556 "region must yield exactly one value (private storage)");
558 return emitOpError(
"yielded value type must match var type");
567void ReductionCombineRegionOp::getSuccessorRegions(
573void ReductionCombineRegionOp::getRegionInvocationBounds(
576 invocationBounds.emplace_back(1, 1);
580ReductionCombineRegionOp::getSuccessorInputs(
RegionSuccessor successor) {
584LogicalResult ReductionCombineRegionOp::verify() {
586 if (
auto yieldOp = dyn_cast<acc::YieldOp>(block.
getTerminator())) {
587 if (yieldOp.getNumOperands() != 0)
588 return emitOpError(
"region must be terminated by acc.yield with no "
598LogicalResult ReductionAccumulateOp::verify() {
599 Type valueType = getValue().getType();
600 auto ptrLikeTy = cast<PointerLikeType>(getMemref().
getType());
601 Type elementType = ptrLikeTy.getElementType();
603 return emitOpError(
"pointer-like destination must have an element type");
604 if (elementType != valueType)
605 return emitOpError(
"pointer-like element type must match value type");
606 if (getParDims().getArray().empty())
607 return emitOpError(
"par_dims must specify at least one parallel dimension");
615LogicalResult ReductionAccumulateArrayOp::verify() {
616 if (getParDims().getArray().empty())
617 return emitOpError(
"par_dims must specify at least one parallel dimension");
625void ReductionCombineOp::getEffects(
641 GPUParallelDimAttr parDim) {
642 for (
auto launchArg : op.getLaunchArgs()) {
643 auto parOp = launchArg.getDefiningOp<ParWidthOp>();
646 auto launchArgDim = cast<GPUParallelDimAttr>(parOp.getParDim());
647 if (launchArgDim == parDim)
653std::optional<Value> ComputeRegionOp::getLaunchArg(GPUParallelDimAttr parDim) {
655 return parWidthOp.getResult();
660ComputeRegionOp::getKnownLaunchArg(GPUParallelDimAttr parDim) {
662 if (parWidthOp.getLaunchArg())
663 return parWidthOp.getLaunchArg();
667std::optional<uint64_t>
668ComputeRegionOp::getKnownConstantLaunchArg(GPUParallelDimAttr parDim) {
669 auto knownParWidth = getKnownLaunchArg(parDim);
670 if (!knownParWidth.has_value())
673 if (!width || *width < 0)
675 return static_cast<uint64_t
>(*width);
679 getInputArgsMutable().append(value);
680 return getBody()->addArgument(value.
getType(), getLoc());
683std::optional<BlockArgument>
684ComputeRegionOp::wireHoistedValueThroughIns(
Value value) {
685 Region ®ion = getRegion();
687 auto useIsInRegion = [&](
OpOperand &use) ->
bool {
692 !llvm::any_of(value.
getUses(), useIsInRegion))
700bool ComputeRegionOp::isEffectivelySerial() {
703 if (getLaunchArg(GPUParallelDimAttr::seqDim(ctx)))
706 auto checkDim = [&](GPUParallelDimAttr dim) ->
bool {
707 auto val = getKnownConstantLaunchArg(dim);
708 return val && *val == 1;
711 return checkDim(GPUParallelDimAttr::threadXDim(ctx)) &&
712 checkDim(GPUParallelDimAttr::threadYDim(ctx)) &&
713 checkDim(GPUParallelDimAttr::threadZDim(ctx)) &&
714 checkDim(GPUParallelDimAttr::blockXDim(ctx)) &&
715 checkDim(GPUParallelDimAttr::blockYDim(ctx)) &&
716 checkDim(GPUParallelDimAttr::blockZDim(ctx));
719bool ComputeRegionOp::launchesMultipleBlocks() {
720 return llvm::any_of(getLaunchParDims(), [&](GPUParallelDimAttr dim) {
721 if (!dim.isAnyBlock())
723 std::optional<uint64_t> width = getKnownConstantLaunchArg(dim);
724 return !width || *width != 1;
728BlockArgument ComputeRegionOp::parDimToWidth(GPUParallelDimAttr parDim) {
729 for (
auto [pos, launchArg] : llvm::enumerate(getLaunchArgs())) {
730 auto parOp = launchArg.getDefiningOp<ParWidthOp>();
732 auto launchArgDim = cast<GPUParallelDimAttr>(parOp.getParDim());
733 if (launchArgDim == parDim) {
734 assert(pos < getRegion().front().getNumArguments() &&
735 "launch arg position out of range");
736 return getRegion().front().getArgument(pos);
739 llvm_unreachable(
"attempting to get unspecified parDim");
744 for (
auto launchArg : getLaunchArgs()) {
745 auto parOp = launchArg.getDefiningOp<ParWidthOp>();
746 auto launchArgDim = cast<GPUParallelDimAttr>(parOp.getParDim());
747 int64_t dimInt = launchArgDim.getValue().getInt();
748 parDims.push_back(intToParDim(
getContext(), dimInt));
754 Block *body = getBody();
758 unsigned numLaunchArgs = getLaunchArgs().size();
759 unsigned numInputArgs = getInputArgs().size();
760 if (argNumber >= numLaunchArgs + numInputArgs)
762 if (argNumber < numLaunchArgs)
763 return getLaunchArgs()[argNumber];
764 return getInputArgs()[argNumber - numLaunchArgs];
767std::optional<BlockArgument> ComputeRegionOp::getBlockArg(
Value value) {
768 Block *body = getBody();
769 for (
auto [idx, launchVal] : llvm::enumerate(getLaunchArgs())) {
770 if (launchVal == value)
773 unsigned numLaunch = getLaunchArgs().size();
774 for (
auto [idx, inputVal] : llvm::enumerate(getInputArgs())) {
775 if (inputVal == value)
783 results.
add<ComputeRegionRemoveDuplicateArgs, ComputeRegionRemoveUnusedArgs>(
787BlockArgument ComputeRegionOp::gpuParWidth(gpu::Processor processor) {
788 return parDimToWidth(GPUParallelDimAttr::get(
getContext(), processor));
791LogicalResult ComputeRegionOp::verify() {
792 for (
auto op : getLaunchArgs())
793 if (!op.getDefiningOp<acc::ParWidthOp>())
795 "launch arguments must be results of acc.par_width operations");
797 unsigned expectedBlockArgs = getLaunchArgs().size() + getInputArgs().size();
798 unsigned actualBlockArgs = getRegion().front().getNumArguments();
799 if (expectedBlockArgs != actualBlockArgs)
800 return emitOpError(
"expected ")
801 << expectedBlockArgs <<
" block arguments (launch + input), got "
808 ValueRange regionArgs = getBody()->getArguments();
812 assert(regionArgs.size() == (launchArgs.size() + inputArgs.size()) &&
813 "region args mismatch");
816 p <<
" stream(" << getStream() <<
" : " << getStream().getType() <<
")";
819 if (!launchArgs.empty()) {
821 for (
size_t j = 0;
j < launchArgs.size(); ++
j, ++i) {
822 p << regionArgs[i] <<
" = " << launchArgs[
j];
823 if (
j < launchArgs.size() - 1)
828 if (!inputArgs.empty()) {
830 for (
size_t j = 0;
j < inputArgs.size(); ++
j, ++i) {
831 p << regionArgs[i] <<
" = " << inputArgs[
j];
832 if (
j < inputArgs.size() - 1)
836 for (
size_t j = 0;
j < inputArgs.size(); ++
j) {
837 p << inputArgs[
j].getType();
838 if (
j < inputArgs.size() - 1)
846 ComputeRegionOp::printProperties(
getContext(), p, getProperties(),
847 getOperandSegmentSizeAttr());
851ParseResult ComputeRegionOp::parse(
OpAsmParser &parser,
862 bool hasStream =
false;
875 for (
size_t i = 0; i < regionArgs.size(); ++i)
876 types.push_back(indexType);
889 for (
auto [iterArg, type] : llvm::zip_equal(regionArgs, types))
895 ComputeRegionOp::ensureTerminator(*body, parser.
getBuilder(),
898 const size_t numLaunchOperands = launchOperands.size();
899 const size_t numInputOperands = inputOperands.size();
900 assert(numLaunchOperands + numInputOperands == regionArgs.size() &&
901 "compute region args mismatch");
904 {
static_cast<int32_t
>(numLaunchOperands),
905 static_cast<int32_t
>(numInputOperands), hasStream ? 1 : 0});
907 for (
size_t i = 0; i < numLaunchOperands; ++i) {
912 for (
size_t i = numLaunchOperands; i < regionArgs.size(); ++i) {
913 if (parser.
resolveOperand(inputOperands[i - numLaunchOperands], types[i],
924 if (ComputeRegionOp::genericParseProperties(parser, parsedProperties))
926 auto propertyDictionary = dyn_cast_or_null<DictionaryAttr>(parsedProperties);
927 if (parsedProperties && !propertyDictionary)
929 "expected properties dictionary");
931 NamedAttrList properties(propertyDictionary ? propertyDictionary
933 properties.set(ComputeRegionOp::getOperandSegmentSizeAttr(),
934 operandSegmentSizes);
935 propertyDictionary = properties.getDictionary(builder.
getContext());
938 << propertyDictionary <<
" for op " <<
result.name.getStringRef()
941 if (
failed(ComputeRegionOp::setPropertiesFromParsedAttr(
942 result.getOrAddProperties<Properties>(), propertyDictionary,
949 for (StringRef attrName : ComputeRegionOp::getAttributeNames()) {
950 if (
result.attributes.get(attrName))
952 <<
"inherent attribute '" << attrName
953 <<
"' cannot be parsed from attr-dict when strict properties in "
954 "assembly format is enabled";
964LogicalResult GPUSharedMemoryOp::verify() {
965 if (getNumCopies() <= 0)
966 return emitOpError(
"num_copies must be positive");
967 if (getStaticUpperBoundBytes() <= 0)
968 return emitOpError(
"static_upper_bound_bytes must be positive");
970 bool hasScaling =
static_cast<bool>(getDynamicSharedMemoryScalingBytes());
971 bool hasFixed =
static_cast<bool>(getDynamicSharedMemoryFixedBytes());
972 if (hasScaling != hasFixed)
974 "dynamic_shared_memory_scaling_bytes and "
975 "dynamic_shared_memory_fixed_bytes must both be present or both be "
977 if (
auto scalingAttr = getDynamicSharedMemoryScalingBytesAttr())
978 if (scalingAttr.getValue().isNegative())
979 return emitOpError(
"dynamic_shared_memory_scaling_bytes must be "
981 if (
auto fixedAttr = getDynamicSharedMemoryFixedBytesAttr())
982 if (fixedAttr.getValue().isNegative())
983 return emitOpError(
"dynamic_shared_memory_fixed_bytes must be "
986 auto resultTy = cast<MemRefType>(getResult().
getType());
988 dyn_cast_if_present<gpu::AddressSpaceAttr>(resultTy.getMemorySpace());
989 if (!addrSpace || addrSpace.getValue() != gpu::AddressSpace::Workgroup)
990 return emitOpError(
"result memref must use #gpu.address_space<workgroup>");
999LogicalResult PredicateRegionOp::verify() {
1000 if (getRegion().empty())
1001 return emitOpError(
"region needs to have at least one block");
1002 if (getRegion().front().getNumArguments() > 0)
1003 return emitOpError(
"region cannot have any arguments");
1004 if (!getOperation()->getParentOfType<ComputeRegionOp>())
1005 return emitOpError(
"must be nested within an acc.compute_region operation");
1013LogicalResult MapInfoOp::verify() {
1018 return emitOpError(
"varType must capture the element type of var");
1022 if (
getDesc() && getDescKind() == DataDescKind::none)
1023 return emitOpError(
"desc requires a descKind other than none");
1028 !acc::bitEnumContainsAny(getDescKind(), DataDescKind::openacc))
1029 return emitOpError(
"bounds require descKind openacc");
1035 if (constantSize && *constantSize < -1)
1036 return emitOpError(
"size must be -1, 0, or a positive byte count");
1046GPUParallelDimAttr GPUParallelDimAttr::get(
MLIRContext *context,
1047 gpu::Processor proc) {
1048 return processorParDim(context, proc);
1051GPUParallelDimAttr GPUParallelDimAttr::seqDim(
MLIRContext *context) {
1052 return processorParDim(context, gpu::Processor::Sequential);
1055GPUParallelDimAttr GPUParallelDimAttr::threadXDim(
MLIRContext *context) {
1056 return processorParDim(context, gpu::Processor::ThreadX);
1059GPUParallelDimAttr GPUParallelDimAttr::threadYDim(
MLIRContext *context) {
1060 return processorParDim(context, gpu::Processor::ThreadY);
1063GPUParallelDimAttr GPUParallelDimAttr::threadZDim(
MLIRContext *context) {
1064 return processorParDim(context, gpu::Processor::ThreadZ);
1067GPUParallelDimAttr GPUParallelDimAttr::blockXDim(
MLIRContext *context) {
1068 return processorParDim(context, gpu::Processor::BlockX);
1071GPUParallelDimAttr GPUParallelDimAttr::blockYDim(
MLIRContext *context) {
1072 return processorParDim(context, gpu::Processor::BlockY);
1075GPUParallelDimAttr GPUParallelDimAttr::blockZDim(
MLIRContext *context) {
1076 return processorParDim(context, gpu::Processor::BlockZ);
1080 GPUParallelDimAttr dim;
1081 if (parser.
parseLess() || parseProcessorValue(parser, dim) ||
1084 "expected format `<` processor_name `>`");
1090void GPUParallelDimAttr::print(
AsmPrinter &printer)
const {
1092 printProcessorValue(printer, *
this);
1096GPUParallelDimAttr GPUParallelDimAttr::threadDim(
MLIRContext *context,
1098 assert(
index <= 2 &&
"thread dimension index must be 0, 1, or 2");
1101 return threadXDim(context);
1103 return threadYDim(context);
1105 return threadZDim(context);
1107 llvm_unreachable(
"validated thread dimension index");
1110GPUParallelDimAttr GPUParallelDimAttr::blockDim(
MLIRContext *context,
1112 assert(
index <= 2 &&
"block dimension index must be 0, 1, or 2");
1115 return blockXDim(context);
1117 return blockYDim(context);
1119 return blockZDim(context);
1121 llvm_unreachable(
"validated block dimension index");
1124gpu::Processor GPUParallelDimAttr::getProcessor()
const {
1125 return indexToGpuProcessor(getValue().getInt());
1128int GPUParallelDimAttr::getOrder()
const {
1129 return gpuProcessorIndex(getProcessor());
1132GPUParallelDimAttr GPUParallelDimAttr::getOneHigher()
const {
1133 int order = getOrder();
1139GPUParallelDimAttr GPUParallelDimAttr::getOneLower()
const {
1140 int order = getOrder();
1146bool GPUParallelDimAttr::isSeq()
const {
1147 return getProcessor() == gpu::Processor::Sequential;
1149bool GPUParallelDimAttr::isThreadX()
const {
1150 return getProcessor() == gpu::Processor::ThreadX;
1152bool GPUParallelDimAttr::isThreadY()
const {
1153 return getProcessor() == gpu::Processor::ThreadY;
1155bool GPUParallelDimAttr::isThreadZ()
const {
1156 return getProcessor() == gpu::Processor::ThreadZ;
1158bool GPUParallelDimAttr::isBlockX()
const {
1159 return getProcessor() == gpu::Processor::BlockX;
1161bool GPUParallelDimAttr::isBlockY()
const {
1162 return getProcessor() == gpu::Processor::BlockY;
1164bool GPUParallelDimAttr::isBlockZ()
const {
1165 return getProcessor() == gpu::Processor::BlockZ;
1167bool GPUParallelDimAttr::isAnyThread()
const {
1168 return isThreadX() || isThreadY() || isThreadZ();
1170bool GPUParallelDimAttr::isAnyBlock()
const {
1171 return isBlockX() || isBlockY() || isBlockZ();
1178GPUParallelDimsAttr GPUParallelDimsAttr::seq(
MLIRContext *ctx) {
1179 return GPUParallelDimsAttr::get(ctx, {GPUParallelDimAttr::seqDim(ctx)});
1182bool GPUParallelDimsAttr::isSeq()
const {
1183 assert(!getArray().empty() &&
"no par_dims found");
1184 if (getArray().size() == 1) {
1185 auto parDim = dyn_cast<GPUParallelDimAttr>(getArray()[0]);
1186 assert(parDim &&
"expected GPUParallelDimAttr");
1187 return parDim.isSeq();
1192bool GPUParallelDimsAttr::isParallel()
const {
return !isSeq(); }
1194bool GPUParallelDimsAttr::isMultiDim()
const {
return getArray().size() > 1; }
1196bool GPUParallelDimsAttr::hasAnyBlockLevel()
const {
1197 return llvm::any_of(
1198 getArray(), [](
const GPUParallelDimAttr &p) {
return p.isAnyBlock(); });
1201bool GPUParallelDimsAttr::hasOnlyBlockLevel()
const {
1202 return !getArray().empty() &&
1203 llvm::all_of(getArray(), [](
const GPUParallelDimAttr &p) {
1204 return p.isAnyBlock();
1208bool GPUParallelDimsAttr::hasOnlyThreadYLevel()
const {
1209 return !getArray().empty() &&
1210 llvm::all_of(getArray(), [](
const GPUParallelDimAttr &p) {
1211 return p.isThreadY();
1215bool GPUParallelDimsAttr::hasOnlyThreadXLevel()
const {
1216 return !getArray().empty() &&
1217 llvm::all_of(getArray(), [](
const GPUParallelDimAttr &p) {
1218 return p.isThreadX();
1223 FailureOr<SmallVector<GPUParallelDimAttr>> parDims =
1224 parseGPUParallelDimList(parser);
1227 return GPUParallelDimsAttr::get(parser.
getContext(), *parDims);
1230void GPUParallelDimsAttr::print(
AsmPrinter &printer)
const {
1231 printGPUParallelDimList(printer, getArray());
1239 FailureOr<SmallVector<GPUParallelDimAttr>> parDims =
1240 parseGPUParallelDimList(parser);
1243 return ActiveParDimsAttr::get(parser.
getContext(), *parDims);
1246void ActiveParDimsAttr::print(
AsmPrinter &printer)
const {
1247 printGPUParallelDimList(printer, getArray());
static void addOperandEffect(SmallVectorImpl< SideEffects::EffectInstance< MemoryEffects::Effect > > &effects, MutableOperandRange operand)
Helper to add an effect on an operand, referenced by its mutable range.
static void addResultEffect(SmallVectorImpl< SideEffects::EffectInstance< MemoryEffects::Effect > > &effects, Value result)
Helper to add an effect on a result value.
static void getSingleRegionOpSuccessorRegions(Operation *op, Region ®ion, RegionBranchPoint point, SmallVectorImpl< RegionSuccessor > ®ions)
Generic helper for single-region OpenACC ops that execute their body once and then continue after the...
static ValueRange getSingleRegionSuccessorInputs(Operation *op, RegionSuccessor successor)
*if copies could not be generated due to yet unimplemented cases *copyInPlacementStart and copyOutPlacementStart in copyPlacementBlock *specify the insertion points where the incoming copies and outgoing should be inserted(the insertion happens right before the *insertion point). Since `begin` can itself be invalidated due to the memref *rewriting done from this method
static ParWidthOp getParWidthOpForLaunchArg(ComputeRegionOp op, GPUParallelDimAttr parDim)
static bool extractWaitClause(ComputeConstructT computeConstruct, DeviceType clauseDeviceType, MLIRContext *context, std::optional< Value > &waitDevnum, SmallVectorImpl< Value > &waitOperands, UnitAttr &waitOnly)
Extract wait for clauseDeviceType. Returns true if a clause was found.
static bool extractAsyncClause(ComputeConstructT computeConstruct, DeviceType clauseDeviceType, MLIRContext *context, std::optional< Value > &asyncOperand, UnitAttr &asyncOnly)
Extract async for clauseDeviceType. Returns true if a clause was found.
static void populateKernelEnvironmentAsyncWait(ComputeConstructT computeConstruct, DeviceType deviceType, std::optional< Value > &asyncOperand, UnitAttr &asyncOnly, std::optional< Value > &waitDevnum, SmallVectorImpl< Value > &waitOperands, UnitAttr &waitOnly)
This base class exposes generic asm parser hooks, usable across the various derived parsers.
@ Square
Square brackets surrounding zero or more operands.
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
virtual ParseResult parseRParen()=0
Parse a ) token.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
ParseResult parseKeywordOrString(std::string *result)
Parse a keyword or a quoted string.
virtual ParseResult parseLess()=0
Parse a '<' token.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual ParseResult parseColon()=0
Parse a : token.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseGreater()=0
Parse a '>' token.
virtual ParseResult parseLParen()=0
Parse a ( token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
virtual ParseResult parseOptionalArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an optional arrow followed by a type list.
ParseResult parseTypeList(SmallVectorImpl< Type > &result)
Parse a type list.
This base class exposes generic asm printer hooks, usable across the various derived printers.
void printOptionalArrowTypeList(TypeRange &&types)
Print an optional arrow followed by a type list.
Attributes are known-constant values of operations.
This class represents an argument of a Block.
unsigned getArgNumber() const
Returns the number of this argument.
Block * getOwner() const
Returns the block that owns this argument.
Block represents an ordered list of Operations.
BlockArgument getArgument(unsigned i)
unsigned getNumArguments()
Operation * getTerminator()
Get the terminator operation of this block.
void eraseArgument(unsigned index)
Erase the argument at 'index' and remove it from the argument list.
DenseI32ArrayAttr getDenseI32ArrayAttr(ArrayRef< int32_t > values)
StringAttr getStringAttr(const Twine &bytes)
MLIRContext * getContext() const
DictionaryAttr getDictionaryAttr(ArrayRef< NamedAttribute > value)
MLIRContext is the top-level object for a collection of MLIR operations.
This class provides a mutable adaptor for a range of operands.
unsigned size() const
Returns the current size of the range.
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult parseRegion(Region ®ion, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region.
ParseResult parseAssignmentList(SmallVectorImpl< Argument > &lhs, SmallVectorImpl< UnresolvedOperand > &rhs)
Parse a list of assignments of the form (x1 = y1, x2 = y2, ...)
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
This class helps build Operations.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
result_range getResults()
Region * getParentRegion()
Returns the region to which the instruction belongs.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
This class represents a point being branched from in the methods of the RegionBranchOpInterface.
bool isParent() const
Returns true if branching from the parent op.
This class represents a successor of a region.
bool isOperation() const
Return true if the successor is an operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
bool isAncestor(Region *other)
Return true if this region is ancestor of the other region.
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
void modifyOpInPlace(Operation *root, CallableT &&callable)
This method is a utility wrapper around an in-place modification of an operation.
virtual void replaceAllUsesWith(Value from, Value to)
Find uses of from and replace them with to.
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replace the results of the given (original) op with a new op that is created without verification (re...
virtual bool replaceUsesWithIf(Value from, Value to, function_ref< bool(OpOperand &)> functor, bool *allUsesReplaced=nullptr)
Find uses of from and replace them with to if the functor returns true.
This class represents a specific instance of an effect.
static DerivedEffect * get()
static CurrentDeviceIdResource * get()
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
bool use_empty() const
Returns true if this value has no uses.
Type getType() const
Return the type of this value.
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Operation * getOwner() const
Return the owner of this operand.
mlir::Value getAccVar(mlir::Operation *accDataClauseOp)
Used to obtain the accVar from a data clause operation.
mlir::Value getVar(mlir::Operation *accDataClauseOp)
Used to obtain the var from a data clause operation.
std::optional< mlir::acc::DataClause > getDataClause(mlir::Operation *accDataEntryOp)
Used to obtain the dataClause from a data entry operation.
mlir::SmallVector< mlir::Value > getBounds(mlir::Operation *accDataClauseOp)
Used to obtain bounds from an acc data clause operation.
Value getDesc(Operation *mapEntryOp)
Returns descriptor value from acc.map_info.
mlir::ArrayAttr getAsyncOnly(mlir::Operation *accDataClauseOp)
Returns an array of acc:DeviceTypeAttr attributes attached to an acc data clause operation,...
mlir::Type getVarType(mlir::Operation *accDataClauseOp)
Used to obtains the varType from a data clause operation which records the type of variable.
Include the generated interface declarations.
void replaceAllUsesInRegionWith(Value orig, Value replacement, Region ®ion)
Replace all uses of orig within the given region with replacement.
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
detail::DenseArrayAttrImpl< int32_t > DenseI32ArrayAttr
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool areValuesDefinedAbove(Range values, Region &limit)
Check if all values in the provided range are defined above the limit region.
This is the representation of an operand reference.
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
This represents an operation in an abstracted form, suitable for use with the builder APIs.
Eliminates variable at the specified position using Fourier-Motzkin variable elimination.