19 #include "llvm/ADT/SmallBitVector.h" 20 #include "llvm/ADT/TypeSwitch.h" 21 #include "llvm/Support/Debug.h" 25 #define DEBUG_TYPE "affine-analysis" 27 #include "mlir/Dialect/Affine/IR/AffineOpsDialect.cpp.inc" 35 return arg.getParentRegion() == region;
59 return legalityCheck(mapping.
lookup(value), dest);
77 return llvm::all_of(values, [&](
Value v) {
84 template <
typename OpTy>
89 "only ops with affine read/write interface are supported");
96 dimOperands, src, dest, mapping,
100 symbolOperands, src, dest, mapping,
111 bool LLVM_ATTRIBUTE_UNUSED
117 op.getMapOperands(), src, dest, mapping,
122 op.getMapOperands(), src, dest, mapping,
149 if (!isa<AffineParallelOp, AffineForOp, AffineIfOp>(destOp))
154 if (!llvm::hasSingleElement(*src))
162 if (
auto iface = dyn_cast<MemoryEffectOpInterface>(op)) {
163 if (iface.hasNoEffect())
197 isa<AffineForOp, AffineParallelOp, AffineIfOp>(parentOp);
201 bool shouldAnalyzeRecursively(
Operation *op)
const final {
return true; }
209 void AffineDialect::initialize() {
212 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc" 214 addInterfaces<AffineInlinerInterface>();
222 return builder.
create<arith::ConstantOp>(loc, type,
value);
246 while (
auto *parentOp = curOp->getParentOp()) {
271 isa<AffineForOp, AffineParallelOp>(parentOp));
293 return isa<AffineForOp, AffineParallelOp>(parentOp);
297 if (
auto applyOp = dyn_cast<AffineApplyOp>(op))
298 return applyOp.isValidDim(region);
301 if (
auto dimOp = dyn_cast<memref::DimOp>(op))
303 if (
auto dimOp = dyn_cast<tensor::DimOp>(op))
311 template <
typename AnyMemRefDefOp>
314 auto memRefType = memrefDefOp.getType();
316 if (!memRefType.isDynamicDim(index))
319 unsigned dynamicDimPos = memRefType.getDynamicDimIndex(index);
320 return isValidSymbol(*(memrefDefOp.getDynamicSizes().begin() + dynamicDimPos),
325 template <
typename OpTy>
333 if (dimOp.getSource().template isa<BlockArgument>())
339 assert(index.hasValue() &&
340 "expect only `dim` operations with a constant index");
341 int64_t i = index.getValue();
343 .Case<memref::ViewOp, memref::SubViewOp, memref::AllocOp>(
345 .Default([](
Operation *) {
return false; });
411 if (
auto applyOp = dyn_cast<AffineApplyOp>(defOp))
412 return applyOp.isValidSymbol(region);
415 if (
auto dimOp = dyn_cast<memref::DimOp>(defOp))
417 if (
auto dimOp = dyn_cast<tensor::DimOp>(defOp))
441 printer <<
'(' << operands.take_front(numDims) <<
')';
442 if (operands.size() > numDims)
443 printer <<
'[' << operands.drop_front(numDims) <<
']';
454 numDims = opInfos.size();
468 template <
typename OpTy>
473 for (
auto operand : operands) {
474 if (opIt++ < numDims) {
476 return op.emitOpError(
"operand cannot be used as a dimension id");
478 return op.emitOpError(
"operand cannot be used as a symbol");
489 return AffineValueMap(getAffineMap(), getOperands(), getResult());
496 AffineMapAttr mapAttr;
502 auto map = mapAttr.getValue();
504 if (map.getNumDims() != numDims ||
505 numDims + map.getNumSymbols() != result.
operands.size()) {
507 "dimension or symbol index mismatch");
510 result.
types.append(map.getNumResults(), indexTy);
515 p <<
" " << getMapAttr();
517 getAffineMap().getNumDims(), p);
528 "operand count and affine map dimension and symbol count must match");
532 return emitOpError(
"mapping must produce one value");
540 return llvm::all_of(getOperands(),
548 return llvm::all_of(getOperands(),
555 return llvm::all_of(getOperands(),
562 return llvm::all_of(getOperands(), [&](
Value operand) {
568 auto map = getAffineMap();
571 auto expr = map.getResult(0);
573 return getOperand(dim.getPosition());
575 return getOperand(map.getNumDims() + sym.getPosition());
578 SmallVector<Attribute, 1> result;
579 if (
failed(map.constantFold(operands, result)))
595 unsigned dimOrSymbolPosition,
598 bool isDimReplacement = (dimOrSymbolPosition < dims.size());
599 unsigned pos = isDimReplacement ? dimOrSymbolPosition
600 : dimOrSymbolPosition - dims.size();
601 Value &v = isDimReplacement ? dims[pos] : syms[pos];
614 AffineMap composeMap = affineApply.getAffineMap();
615 assert(composeMap.
getNumResults() == 1 &&
"affine.apply with >1 results");
617 composeMap.
shiftDims(dims.size()).shiftSymbols(syms.size()).getResult(0);
619 affineApply.getMapOperands().take_front(composeMap.
getNumDims());
621 affineApply.getMapOperands().take_back(composeMap.
getNumSymbols());
627 dims.append(composeDims.begin(), composeDims.end());
628 syms.append(composeSyms.begin(), composeSyms.end());
629 *map = map->
replace(toReplace, composeExpr, dims.size(), syms.size());
646 SmallVector<Value, 4> dims(operands->begin(),
648 SmallVector<Value, 4> syms(operands->begin() + map->
getNumDims(),
657 bool changed =
false;
658 for (
unsigned pos = 0; pos != dims.size() + syms.size(); ++pos)
670 unsigned nDims = 0, nSyms = 0;
671 SmallVector<AffineExpr, 4> dimReplacements, symReplacements;
672 dimReplacements.reserve(dims.size());
673 symReplacements.reserve(syms.size());
674 for (
auto *container : {&dims, &syms}) {
675 bool isDim = (container == &dims);
676 auto &repls = isDim ? dimReplacements : symReplacements;
678 Value v = en.value();
682 "map is function of unexpected expr@pos");
688 operands->push_back(v);
701 while (llvm::any_of(*operands, [](
Value v) {
714 assert(normalizedMap);
715 return b.
create<AffineApplyOp>(loc, normalizedMap, normalizedOperands);
730 SmallVector<Value, 4> operands(operandsRef.begin(), operandsRef.end());
733 return b.
createOrFold<AffineApplyOp>(loc, map, operands);
753 template <
class MapOrSet>
756 if (!mapOrSet || operands->empty())
759 assert(mapOrSet->getNumInputs() == operands->size() &&
760 "map/set inputs must match number of operands");
762 auto *context = mapOrSet->getContext();
763 SmallVector<Value, 8> resultOperands;
764 resultOperands.reserve(operands->size());
765 SmallVector<Value, 8> remappedSymbols;
766 remappedSymbols.reserve(operands->size());
767 unsigned nextDim = 0;
768 unsigned nextSym = 0;
769 unsigned oldNumSyms = mapOrSet->getNumSymbols();
770 SmallVector<AffineExpr, 8> dimRemapping(mapOrSet->getNumDims());
771 for (
unsigned i = 0, e = mapOrSet->getNumInputs(); i != e; ++i) {
772 if (i < mapOrSet->getNumDims()) {
776 remappedSymbols.push_back((*operands)[i]);
779 resultOperands.push_back((*operands)[i]);
782 resultOperands.push_back((*operands)[i]);
786 resultOperands.append(remappedSymbols.begin(), remappedSymbols.end());
787 *operands = resultOperands;
788 *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, {}, nextDim,
789 oldNumSyms + nextSym);
791 assert(mapOrSet->getNumInputs() == operands->size() &&
792 "map/set inputs must match number of operands");
796 template <
class MapOrSet>
800 "Argument must be either of AffineMap or IntegerSet type");
802 if (!mapOrSet || operands->empty())
805 assert(mapOrSet->getNumInputs() == operands->size() &&
806 "map/set inputs must match number of operands");
808 canonicalizePromotedSymbols<MapOrSet>(mapOrSet, operands);
811 llvm::SmallBitVector usedDims(mapOrSet->getNumDims());
812 llvm::SmallBitVector usedSyms(mapOrSet->getNumSymbols());
815 usedDims[dimExpr.getPosition()] =
true;
817 usedSyms[symExpr.getPosition()] =
true;
820 auto *context = mapOrSet->getContext();
822 SmallVector<Value, 8> resultOperands;
823 resultOperands.reserve(operands->size());
825 llvm::SmallDenseMap<Value, AffineExpr, 8> seenDims;
826 SmallVector<AffineExpr, 8> dimRemapping(mapOrSet->getNumDims());
827 unsigned nextDim = 0;
828 for (
unsigned i = 0, e = mapOrSet->getNumDims(); i != e; ++i) {
831 auto it = seenDims.find((*operands)[i]);
832 if (it == seenDims.end()) {
834 resultOperands.push_back((*operands)[i]);
835 seenDims.insert(std::make_pair((*operands)[i], dimRemapping[i]));
837 dimRemapping[i] = it->second;
841 llvm::SmallDenseMap<Value, AffineExpr, 8> seenSymbols;
842 SmallVector<AffineExpr, 8> symRemapping(mapOrSet->getNumSymbols());
843 unsigned nextSym = 0;
844 for (
unsigned i = 0, e = mapOrSet->getNumSymbols(); i != e; ++i) {
850 IntegerAttr operandCst;
851 if (
matchPattern((*operands)[i + mapOrSet->getNumDims()],
858 auto it = seenSymbols.find((*operands)[i + mapOrSet->getNumDims()]);
859 if (it == seenSymbols.end()) {
861 resultOperands.push_back((*operands)[i + mapOrSet->getNumDims()]);
862 seenSymbols.insert(std::make_pair((*operands)[i + mapOrSet->getNumDims()],
865 symRemapping[i] = it->second;
868 *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, symRemapping,
870 *operands = resultOperands;
875 canonicalizeMapOrSetAndOperands<AffineMap>(map, operands);
880 canonicalizeMapOrSetAndOperands<IntegerSet>(
set, operands);
887 template <
typename AffineOpTy>
899 llvm::is_one_of<AffineOpTy, AffineLoadOp, AffinePrefetchOp,
900 AffineStoreOp, AffineApplyOp, AffineMinOp, AffineMaxOp,
901 AffineVectorStoreOp, AffineVectorLoadOp>::value,
902 "affine load/store/vectorstore/vectorload/apply/prefetch/min/max op " 904 auto map = affineOp.getAffineMap();
906 auto oldOperands = affineOp.getMapOperands();
910 if (map == oldMap && std::equal(oldOperands.begin(), oldOperands.end(),
911 resultOperands.begin()))
914 replaceAffineOp(rewriter, affineOp, map, resultOperands);
922 void SimplifyAffineOp<AffineLoadOp>::replaceAffineOp(
929 void SimplifyAffineOp<AffinePrefetchOp>::replaceAffineOp(
933 prefetch, prefetch.getMemref(), map, mapOperands,
934 prefetch.getLocalityHint(), prefetch.getIsWrite(),
935 prefetch.getIsDataCache());
938 void SimplifyAffineOp<AffineStoreOp>::replaceAffineOp(
942 store, store.getValueToStore(), store.getMemRef(), map, mapOperands);
945 void SimplifyAffineOp<AffineVectorLoadOp>::replaceAffineOp(
949 vectorload, vectorload.getVectorType(), vectorload.getMemRef(), map,
953 void SimplifyAffineOp<AffineVectorStoreOp>::replaceAffineOp(
957 vectorstore, vectorstore.getValueToStore(), vectorstore.getMemRef(), map,
962 template <
typename AffineOpTy>
963 void SimplifyAffineOp<AffineOpTy>::replaceAffineOp(
972 results.
add<SimplifyAffineOp<AffineApplyOp>>(context);
985 auto cast = operand.get().getDefiningOp<memref::CastOp>();
986 if (cast && operand.get() != ignore &&
988 operand.set(cast.getOperand());
1008 result.
addAttribute(getSrcMapAttrStrName(), AffineMapAttr::get(srcMap));
1011 result.
addAttribute(getDstMapAttrStrName(), AffineMapAttr::get(dstMap));
1014 result.
addAttribute(getTagMapAttrStrName(), AffineMapAttr::get(tagMap));
1023 p <<
" " << getSrcMemRef() <<
'[';
1025 p <<
"], " << getDstMemRef() <<
'[';
1027 p <<
"], " << getTagMemRef() <<
'[';
1031 p <<
", " << getStride();
1032 p <<
", " << getNumElementsPerStride();
1034 p <<
" : " << getSrcMemRefType() <<
", " << getDstMemRefType() <<
", " 1035 << getTagMemRefType();
1047 AffineMapAttr srcMapAttr;
1050 AffineMapAttr dstMapAttr;
1053 AffineMapAttr tagMapAttr;
1068 getSrcMapAttrStrName(),
1072 getDstMapAttrStrName(),
1076 getTagMapAttrStrName(),
1085 if (!strideInfo.empty() && strideInfo.size() != 2) {
1087 "expected two stride related operands");
1089 bool isStrided = strideInfo.size() == 2;
1094 if (types.size() != 3)
1112 if (srcMapOperands.size() != srcMapAttr.getValue().getNumInputs() ||
1113 dstMapOperands.size() != dstMapAttr.getValue().getNumInputs() ||
1114 tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1116 "memref operand count not equal to map.numInputs");
1121 if (!getOperand(getSrcMemRefOperandIndex()).getType().isa<MemRefType>())
1122 return emitOpError(
"expected DMA source to be of memref type");
1123 if (!getOperand(getDstMemRefOperandIndex()).getType().isa<MemRefType>())
1124 return emitOpError(
"expected DMA destination to be of memref type");
1125 if (!getOperand(getTagMemRefOperandIndex()).getType().isa<MemRefType>())
1126 return emitOpError(
"expected DMA tag to be of memref type");
1128 unsigned numInputsAllMaps = getSrcMap().getNumInputs() +
1129 getDstMap().getNumInputs() +
1130 getTagMap().getNumInputs();
1131 if (getNumOperands() != numInputsAllMaps + 3 + 1 &&
1132 getNumOperands() != numInputsAllMaps + 3 + 1 + 2) {
1133 return emitOpError(
"incorrect number of operands");
1137 for (
auto idx : getSrcIndices()) {
1138 if (!idx.getType().isIndex())
1139 return emitOpError(
"src index to dma_start must have 'index' type");
1141 return emitOpError(
"src index must be a dimension or symbol identifier");
1143 for (
auto idx : getDstIndices()) {
1144 if (!idx.getType().isIndex())
1145 return emitOpError(
"dst index to dma_start must have 'index' type");
1147 return emitOpError(
"dst index must be a dimension or symbol identifier");
1149 for (
auto idx : getTagIndices()) {
1150 if (!idx.getType().isIndex())
1151 return emitOpError(
"tag index to dma_start must have 'index' type");
1153 return emitOpError(
"tag index must be a dimension or symbol identifier");
1173 result.
addAttribute(getTagMapAttrStrName(), AffineMapAttr::get(tagMap));
1179 p <<
" " << getTagMemRef() <<
'[';
1184 p <<
" : " << getTagMemRef().getType();
1195 AffineMapAttr tagMapAttr;
1204 getTagMapAttrStrName(),
1213 if (!type.
isa<MemRefType>())
1215 "expected tag to be of memref type");
1217 if (tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1219 "tag memref operand count != to map.numInputs");
1224 if (!getOperand(0).getType().isa<MemRefType>())
1225 return emitOpError(
"expected DMA tag to be of memref type");
1227 for (
auto idx : getTagIndices()) {
1228 if (!idx.getType().isIndex())
1229 return emitOpError(
"index to dma_wait must have 'index' type");
1231 return emitOpError(
"index must be a dimension or symbol identifier");
1251 ValueRange iterArgs, BodyBuilderFn bodyBuilder) {
1252 assert(((!lbMap && lbOperands.empty()) ||
1254 "lower bound operand count does not match the affine map");
1255 assert(((!ubMap && ubOperands.empty()) ||
1257 "upper bound operand count does not match the affine map");
1258 assert(step > 0 &&
"step has to be a positive integer constant");
1260 for (
Value val : iterArgs)
1268 result.
addAttribute(getLowerBoundAttrStrName(), AffineMapAttr::get(lbMap));
1272 result.
addAttribute(getUpperBoundAttrStrName(), AffineMapAttr::get(ubMap));
1281 Value inductionVar =
1283 for (
Value val : iterArgs)
1284 bodyBlock.
addArgument(val.getType(), val.getLoc());
1289 if (iterArgs.empty() && !bodyBuilder) {
1290 ensureTerminator(*bodyRegion, builder, result.
location);
1291 }
else if (bodyBuilder) {
1294 bodyBuilder(builder, result.
location, inductionVar,
1300 int64_t ub, int64_t step,
ValueRange iterArgs,
1301 BodyBuilderFn bodyBuilder) {
1304 return build(builder, result, {}, lbMap, {}, ubMap, step, iterArgs,
1311 auto *body = getBody();
1312 if (body->getNumArguments() == 0 || !body->getArgument(0).getType().isIndex())
1313 return emitOpError(
"expected body to have a single index argument for the " 1314 "induction variable");
1318 if (getLowerBoundMap().getNumInputs() > 0)
1320 getLowerBoundMap().getNumDims())))
1323 if (getUpperBoundMap().getNumInputs() > 0)
1325 getUpperBoundMap().getNumDims())))
1328 unsigned opNumResults = getNumResults();
1329 if (opNumResults == 0)
1335 if (getNumIterOperands() != opNumResults)
1337 "mismatch between the number of loop-carried values and results");
1338 if (getNumRegionIterArgs() != opNumResults)
1340 "mismatch between the number of basic block args and results");
1350 bool failedToParsedMinMax =
1354 auto boundAttrStrName = isLower ? AffineForOp::getLowerBoundAttrStrName()
1355 : AffineForOp::getUpperBoundAttrStrName();
1358 SmallVector<OpAsmParser::UnresolvedOperand, 1> boundOpInfos;
1362 if (!boundOpInfos.empty()) {
1364 if (boundOpInfos.size() > 1)
1366 "expected only one loop bound operand");
1370 if (p.
resolveOperand(boundOpInfos.front(), builder.getIndexType(),
1377 AffineMap map = builder.getSymbolIdentityMap();
1378 result.
addAttribute(boundAttrStrName, AffineMapAttr::get(map));
1386 if (p.
parseAttribute(boundAttr, builder.getIndexType(), boundAttrStrName,
1391 if (
auto affineMapAttr = boundAttr.
dyn_cast<AffineMapAttr>()) {
1392 unsigned currentNumOperands = result.
operands.size();
1397 auto map = affineMapAttr.getValue();
1398 if (map.getNumDims() != numDims)
1401 "dim operand count and affine map dim count must match");
1403 unsigned numDimAndSymbolOperands =
1404 result.
operands.size() - currentNumOperands;
1405 if (numDims + map.getNumSymbols() != numDimAndSymbolOperands)
1408 "symbol operand count and affine map symbol count must match");
1412 if (map.getNumResults() > 1 && failedToParsedMinMax) {
1414 return p.
emitError(attrLoc,
"lower loop bound affine map with " 1415 "multiple results requires 'max' prefix");
1417 return p.
emitError(attrLoc,
"upper loop bound affine map with multiple " 1418 "results requires 'min' prefix");
1424 if (
auto integerAttr = boundAttr.
dyn_cast<IntegerAttr>()) {
1428 AffineMapAttr::get(builder.getConstantAffineMap(integerAttr.getInt())));
1434 "expected valid affine map representation for loop bounds");
1454 AffineForOp::getStepAttrStrName(),
1458 IntegerAttr stepAttr;
1460 AffineForOp::getStepAttrStrName().data(),
1464 if (stepAttr.getValue().getSExtValue() < 0)
1467 "expected step to be representable as a positive signed integer");
1471 SmallVector<OpAsmParser::Argument, 4> regionArgs;
1472 SmallVector<OpAsmParser::UnresolvedOperand, 4> operands;
1475 regionArgs.push_back(inductionVariable);
1483 for (
auto argOperandType :
1484 llvm::zip(llvm::drop_begin(regionArgs), operands, result.
types)) {
1485 Type type = std::get<2>(argOperandType);
1486 std::get<0>(argOperandType).type = type;
1495 if (regionArgs.size() != result.
types.size() + 1)
1498 "mismatch between the number of loop-carried values and results");
1502 AffineForOp::ensureTerminator(*body, builder, result.
location);
1525 p << constExpr.getValue();
1549 unsigned AffineForOp::getNumIterOperands() {
1550 AffineMap lbMap = getLowerBoundMapAttr().getValue();
1551 AffineMap ubMap = getUpperBoundMapAttr().getValue();
1566 p <<
" step " << getStep();
1568 bool printBlockTerminators =
false;
1569 if (getNumIterOperands() > 0) {
1571 auto regionArgs = getRegionIterArgs();
1572 auto operands = getIterOperands();
1574 llvm::interleaveComma(llvm::zip(regionArgs, operands), p, [&](
auto it) {
1575 p << std::get<0>(it) <<
" = " << std::get<1>(it);
1577 p <<
") -> (" << getResultTypes() <<
")";
1578 printBlockTerminators =
true;
1583 printBlockTerminators);
1585 {getLowerBoundAttrStrName(),
1586 getUpperBoundAttrStrName(),
1587 getStepAttrStrName()});
1592 auto foldLowerOrUpperBound = [&forOp](
bool lower) {
1595 SmallVector<Attribute, 8> operandConstants;
1596 auto boundOperands =
1597 lower ? forOp.getLowerBoundOperands() : forOp.getUpperBoundOperands();
1598 for (
auto operand : boundOperands) {
1601 operandConstants.push_back(operandCst);
1605 lower ? forOp.getLowerBoundMap() : forOp.getUpperBoundMap();
1607 "bound maps should have at least one result");
1608 SmallVector<Attribute, 4> foldedResults;
1613 assert(!foldedResults.empty() &&
"bounds should have at least one result");
1614 auto maxOrMin = foldedResults[0].cast<IntegerAttr>().getValue();
1615 for (
unsigned i = 1, e = foldedResults.size(); i < e; i++) {
1616 auto foldedResult = foldedResults[i].cast<IntegerAttr>().getValue();
1617 maxOrMin = lower ? llvm::APIntOps::smax(maxOrMin, foldedResult)
1618 : llvm::APIntOps::smin(maxOrMin, foldedResult);
1620 lower ? forOp.setConstantLowerBound(maxOrMin.getSExtValue())
1621 : forOp.setConstantUpperBound(maxOrMin.getSExtValue());
1626 bool folded =
false;
1627 if (!forOp.hasConstantLowerBound())
1628 folded |=
succeeded(foldLowerOrUpperBound(
true));
1631 if (!forOp.hasConstantUpperBound())
1632 folded |=
succeeded(foldLowerOrUpperBound(
false));
1638 SmallVector<Value, 4> lbOperands(forOp.getLowerBoundOperands());
1639 SmallVector<Value, 4> ubOperands(forOp.getUpperBoundOperands());
1641 auto lbMap = forOp.getLowerBoundMap();
1642 auto ubMap = forOp.getUpperBoundMap();
1643 auto prevLbMap = lbMap;
1644 auto prevUbMap = ubMap;
1655 if (lbMap == prevLbMap && ubMap == prevUbMap)
1658 if (lbMap != prevLbMap)
1659 forOp.setLowerBound(lbOperands, lbMap);
1660 if (ubMap != prevUbMap)
1661 forOp.setUpperBound(ubOperands, ubMap);
1668 int64_t step = forOp.getStep();
1669 if (!forOp.hasConstantBounds() || step <= 0)
1671 int64_t lb = forOp.getConstantLowerBound();
1672 int64_t ub = forOp.getConstantUpperBound();
1673 return ub - lb <= 0 ? 0 : (ub - lb + step - 1) / step;
1684 if (!llvm::hasSingleElement(*forOp.getBody()))
1686 if (forOp.getNumResults() == 0)
1689 if (tripCount && *tripCount == 0) {
1692 rewriter.
replaceOp(forOp, forOp.getIterOperands());
1695 SmallVector<Value, 4> replacements;
1696 auto yieldOp = cast<AffineYieldOp>(forOp.getBody()->getTerminator());
1697 auto iterArgs = forOp.getRegionIterArgs();
1698 bool hasValDefinedOutsideLoop =
false;
1699 bool iterArgsNotInOrder =
false;
1700 for (
unsigned i = 0, e = yieldOp->getNumOperands(); i < e; ++i) {
1701 Value val = yieldOp.getOperand(i);
1702 auto *iterArgIt = llvm::find(iterArgs, val);
1703 if (iterArgIt == iterArgs.end()) {
1705 assert(forOp.isDefinedOutsideOfLoop(val) &&
1706 "must be defined outside of the loop");
1707 hasValDefinedOutsideLoop =
true;
1708 replacements.push_back(val);
1710 unsigned pos = std::distance(iterArgs.begin(), iterArgIt);
1712 iterArgsNotInOrder =
true;
1713 replacements.push_back(forOp.getIterOperands()[pos]);
1718 if (!tripCount.hasValue() &&
1719 (hasValDefinedOutsideLoop || iterArgsNotInOrder))
1723 if (tripCount.hasValue() && tripCount.getValue() >= 2 && iterArgsNotInOrder)
1725 rewriter.
replaceOp(forOp, replacements);
1733 results.
add<AffineForEmptyLoopFolder>(context);
1741 assert(!index || *index == 0 &&
"invalid region index");
1745 return getIterOperands();
1753 void AffineForOp::getSuccessorRegions(
1756 assert((!index.hasValue() || index.getValue() == 0) &&
1757 "expected loop region");
1763 if (!index.hasValue() && tripCount.hasValue()) {
1764 if (tripCount.getValue() > 0) {
1765 regions.push_back(
RegionSuccessor(&getLoopBody(), getRegionIterArgs()));
1768 if (tripCount.getValue() == 0) {
1776 if (index && tripCount && *tripCount == 1) {
1783 regions.push_back(
RegionSuccessor(&getLoopBody(), getRegionIterArgs()));
1790 return tripCount && *tripCount == 0;
1800 results.assign(getIterOperands().begin(), getIterOperands().end());
1807 auto lbMap = getLowerBoundMap();
1812 auto lbMap = getLowerBoundMap();
1813 auto ubMap = getUpperBoundMap();
1820 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
1822 SmallVector<Value, 4> newOperands(lbOperands.begin(), lbOperands.end());
1825 newOperands.append(ubOperands.begin(), ubOperands.end());
1826 auto iterOperands = getIterOperands();
1827 newOperands.append(iterOperands.begin(), iterOperands.end());
1828 (*this)->setOperands(newOperands);
1830 (*this)->setAttr(getLowerBoundAttrStrName(), AffineMapAttr::get(map));
1835 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
1838 newOperands.append(ubOperands.begin(), ubOperands.end());
1839 auto iterOperands = getIterOperands();
1840 newOperands.append(iterOperands.begin(), iterOperands.end());
1841 (*this)->setOperands(newOperands);
1843 (*this)->setAttr(getUpperBoundAttrStrName(), AffineMapAttr::get(map));
1846 void AffineForOp::setLowerBoundMap(
AffineMap map) {
1847 auto lbMap = getLowerBoundMap();
1850 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
1852 (*this)->setAttr(getLowerBoundAttrStrName(), AffineMapAttr::get(map));
1855 void AffineForOp::setUpperBoundMap(
AffineMap map) {
1856 auto ubMap = getUpperBoundMap();
1859 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
1861 (*this)->setAttr(getUpperBoundAttrStrName(), AffineMapAttr::get(map));
1864 bool AffineForOp::hasConstantLowerBound() {
1868 bool AffineForOp::hasConstantUpperBound() {
1869 return getUpperBoundMap().isSingleConstant();
1872 int64_t AffineForOp::getConstantLowerBound() {
1873 return getLowerBoundMap().getSingleConstantResult();
1876 int64_t AffineForOp::getConstantUpperBound() {
1877 return getUpperBoundMap().getSingleConstantResult();
1880 void AffineForOp::setConstantLowerBound(int64_t value) {
1884 void AffineForOp::setConstantUpperBound(int64_t value) {
1889 return {operand_begin(), operand_begin() + getLowerBoundMap().getNumInputs()};
1893 return {operand_begin() + getLowerBoundMap().getNumInputs(),
1894 operand_begin() + getLowerBoundMap().getNumInputs() +
1895 getUpperBoundMap().getNumInputs()};
1898 AffineForOp::operand_range AffineForOp::getControlOperands() {
1899 return {operand_begin(), operand_begin() + getLowerBoundMap().getNumInputs() +
1900 getUpperBoundMap().getNumInputs()};
1903 bool AffineForOp::matchingBoundOperandList() {
1904 auto lbMap = getLowerBoundMap();
1905 auto ubMap = getUpperBoundMap();
1911 for (
unsigned i = 0, e = lbMap.
getNumInputs(); i < e; i++) {
1913 if (getOperand(i) != getOperand(numOperands + i))
1919 Region &AffineForOp::getLoopBody() {
return getRegion(); }
1922 return getInductionVar();
1926 if (!hasConstantLowerBound())
1938 if (!hasConstantUpperBound())
1954 if (!ivArg || !ivArg.getOwner())
1955 return AffineForOp();
1957 if (
auto forOp = dyn_cast<AffineForOp>(containingInst))
1959 return forOp.getInductionVar() == val ? forOp : AffineForOp();
1960 return AffineForOp();
1967 ivs->reserve(forInsts.size());
1968 for (
auto forInst : forInsts)
1969 ivs->push_back(forInst.getInductionVar());
1974 template <
typename BoundListTy,
typename LoopCreatorTy>
1979 LoopCreatorTy &&loopCreatorFn) {
1980 assert(lbs.size() == ubs.size() &&
"Mismatch in number of arguments");
1981 assert(lbs.size() == steps.size() &&
"Mismatch in number of arguments");
1992 SmallVector<Value, 4> ivs;
1993 ivs.reserve(lbs.size());
1994 for (
unsigned i = 0, e = lbs.size(); i < e; ++i) {
2000 if (i == e - 1 && bodyBuilderFn) {
2002 bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
2004 nestedBuilder.
create<AffineYieldOp>(nestedLoc);
2009 auto loop = loopCreatorFn(builder, loc, lbs[i], ubs[i], steps[i], loopBody);
2017 int64_t ub, int64_t step,
2018 AffineForOp::BodyBuilderFn bodyBuilderFn) {
2019 return builder.
create<AffineForOp>(loc, lb, ub, step, llvm::None,
2027 AffineForOp::BodyBuilderFn bodyBuilderFn) {
2030 if (lbConst && ubConst)
2032 ubConst.
value(), step, bodyBuilderFn);
2035 llvm::None, bodyBuilderFn);
2058 bool replaceLoopResults) {
2059 assert(newIterOperands.size() == newYieldedValues.size() &&
2060 "newIterOperands must be of the same size as newYieldedValues");
2064 auto operands = llvm::to_vector<4>(loop.getIterOperands());
2065 operands.append(newIterOperands.begin(), newIterOperands.end());
2069 auto lbMap = loop.getLowerBoundMap();
2070 auto ubMap = loop.getUpperBoundMap();
2071 AffineForOp newLoop =
2072 b.
create<AffineForOp>(loop.getLoc(), lbOperands, lbMap, ubOperands, ubMap,
2073 loop.getStep(), operands);
2075 newLoop.getLoopBody().takeBody(loop.getLoopBody());
2076 for (
Value val : newIterArgs)
2077 newLoop.getLoopBody().addArgument(val.getType(), val.getLoc());
2080 if (!newYieldedValues.empty()) {
2081 auto yield = cast<AffineYieldOp>(newLoop.getBody()->getTerminator());
2083 auto yieldOperands = llvm::to_vector<4>(yield.getOperands());
2084 yieldOperands.append(newYieldedValues.begin(), newYieldedValues.end());
2085 b.
create<AffineYieldOp>(yield.getLoc(), yieldOperands);
2088 if (replaceLoopResults) {
2089 for (
auto it : llvm::zip(loop.getResults(), newLoop.getResults().take_front(
2090 loop.getNumResults()))) {
2091 std::get<0>(it).replaceAllUsesWith(std::get<1>(it));
2108 if (ifOp.getElseRegion().empty() ||
2109 !llvm::hasSingleElement(*ifOp.getElseBlock()) || ifOp.getNumResults())
2127 auto isTriviallyFalse = [](
IntegerSet iSet) {
2128 return iSet.isEmptyIntegerSet();
2132 return (iSet.getNumEqualities() == 1 && iSet.getNumInequalities() == 0 &&
2133 iSet.getConstraint(0) == 0);
2136 IntegerSet affineIfConditions = op.getIntegerSet();
2138 if (isTriviallyFalse(affineIfConditions)) {
2142 if (op.getNumResults() == 0 && !op.hasElse()) {
2148 blockToMove = op.getElseBlock();
2149 }
else if (isTriviallyTrue(affineIfConditions)) {
2150 blockToMove = op.getThenBlock();
2168 rewriter.
eraseOp(blockToMoveTerminator);
2177 auto conditionAttr =
2178 (*this)->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName());
2180 return emitOpError(
"requires an integer set attribute named 'condition'");
2183 IntegerSet condition = conditionAttr.getValue();
2185 return emitOpError(
"operand count and condition integer set dimension and " 2186 "symbol count must match");
2198 IntegerSetAttr conditionAttr;
2201 AffineIfOp::getConditionAttrStrName(),
2207 auto set = conditionAttr.getValue();
2208 if (
set.getNumDims() != numDims)
2211 "dim operand count and integer set dim count must match");
2212 if (numDims +
set.getNumSymbols() != result.
operands.size())
2215 "symbol operand count and integer set symbol count must match");
2229 AffineIfOp::ensureTerminator(*thenRegion, parser.
getBuilder(),
2236 AffineIfOp::ensureTerminator(*elseRegion, parser.
getBuilder(),
2248 auto conditionAttr =
2249 (*this)->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName());
2250 p <<
" " << conditionAttr;
2252 conditionAttr.getValue().getNumDims(), p);
2259 auto &elseRegion = this->getElseRegion();
2260 if (!elseRegion.empty()) {
2269 getConditionAttrStrName());
2274 ->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName())
2278 void AffineIfOp::setIntegerSet(
IntegerSet newSet) {
2279 (*this)->setAttr(getConditionAttrStrName(), IntegerSetAttr::get(newSet));
2284 (*this)->setOperands(operands);
2289 bool withElseRegion) {
2290 assert(resultTypes.empty() || withElseRegion);
2293 result.
addAttribute(getConditionAttrStrName(), IntegerSetAttr::get(
set));
2297 if (resultTypes.empty())
2298 AffineIfOp::ensureTerminator(*thenRegion, builder, result.
location);
2301 if (withElseRegion) {
2303 if (resultTypes.empty())
2304 AffineIfOp::ensureTerminator(*elseRegion, builder, result.
location);
2310 AffineIfOp::build(builder, result, {},
set, args,
2317 auto set = getIntegerSet();
2324 if (operands.size() < getIntegerSet().getNumInputs() ||
2325 set.getNumSymbols() > getIntegerSet().getNumSymbols()) {
2326 setConditional(
set, operands);
2335 results.
add<SimplifyDeadElse, AlwaysTrueOrFalseIf>(context);
2344 assert(operands.size() == 1 + map.
getNumInputs() &&
"inconsistent operands");
2347 result.
addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
2348 auto memrefType = operands[0].
getType().cast<MemRefType>();
2349 result.
types.push_back(memrefType.getElementType());
2354 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
2357 auto memrefType = memref.
getType().
cast<MemRefType>();
2358 result.
addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
2359 result.
types.push_back(memrefType.getElementType());
2364 auto memrefType = memref.
getType().
cast<MemRefType>();
2365 int64_t rank = memrefType.getRank();
2370 build(builder, result, memref, map, indices);
2379 AffineMapAttr mapAttr;
2384 AffineLoadOp::getMapAttrStrName(),
2394 p <<
" " << getMemRef() <<
'[';
2395 if (AffineMapAttr mapAttr =
2396 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
2400 {getMapAttrStrName()});
2409 MemRefType memrefType,
unsigned numIndexOperands) {
2413 return op->
emitOpError(
"affine map num results must equal memref rank");
2415 return op->
emitOpError(
"expects as many subscripts as affine map inputs");
2417 if (memrefType.getRank() != numIndexOperands)
2419 "expects the number of subscripts to be equal to memref rank");
2423 for (
auto idx : mapOperands) {
2424 if (!idx.getType().isIndex())
2425 return op->
emitOpError(
"index to load must have 'index' type");
2427 return op->
emitOpError(
"index must be a dimension or symbol identifier");
2435 if (getType() != memrefType.getElementType())
2436 return emitOpError(
"result type must match element type of memref");
2440 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
2441 getMapOperands(), memrefType,
2442 getNumOperands() - 1)))
2450 results.
add<SimplifyAffineOp<AffineLoadOp>>(context);
2459 auto getGlobalOp = getMemref().getDefiningOp<memref::GetGlobalOp>();
2466 auto global = dyn_cast_or_null<memref::GlobalOp>(
2480 if (!getAffineMap().isConstant())
2482 auto indices = llvm::to_vector<4>(
2483 llvm::map_range(getAffineMap().getConstantResults(),
2484 [](int64_t v) -> uint64_t {
return v; }));
2485 return cstAttr.getValues<
Attribute>()[indices];
2495 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
2499 result.
addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
2506 auto memrefType = memref.
getType().
cast<MemRefType>();
2507 int64_t rank = memrefType.getRank();
2512 build(builder, result, valueToStore, memref, map, indices);
2521 AffineMapAttr mapAttr;
2522 SmallVector<OpAsmParser::UnresolvedOperand, 1> mapOperands;
2526 mapOperands, mapAttr, AffineStoreOp::getMapAttrStrName(),
2537 p <<
" " << getValueToStore();
2538 p <<
", " << getMemRef() <<
'[';
2539 if (AffineMapAttr mapAttr =
2540 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
2544 {getMapAttrStrName()});
2551 if (getValueToStore().getType() != memrefType.getElementType())
2553 "value to store must have the same type as memref element type");
2557 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
2558 getMapOperands(), memrefType,
2559 getNumOperands() - 2)))
2567 results.
add<SimplifyAffineOp<AffineStoreOp>>(context);
2582 if (op.getNumOperands() !=
2583 op.getMap().getNumDims() + op.getMap().getNumSymbols())
2584 return op.emitOpError(
2585 "operand count and affine map dimension and symbol count must match");
2590 p <<
' ' << op->getAttr(T::getMapAttrStrName());
2591 auto operands = op.getOperands();
2592 unsigned numDims = op.getMap().getNumDims();
2593 p <<
'(' << operands.take_front(numDims) <<
')';
2595 if (operands.size() != numDims)
2596 p <<
'[' << operands.drop_front(numDims) <<
']';
2598 {T::getMapAttrStrName()});
2601 template <
typename T>
2606 SmallVector<OpAsmParser::UnresolvedOperand, 8> dimInfos;
2607 SmallVector<OpAsmParser::UnresolvedOperand, 8> symInfos;
2608 AffineMapAttr mapAttr;
2624 template <
typename T>
2627 "expected affine min or max op");
2632 SmallVector<int64_t, 2> results;
2633 auto foldedMap = op.getMap().partialConstantFold(operands, &results);
2636 if (results.empty()) {
2638 if (foldedMap == op.getMap())
2640 op->setAttr(
"map", AffineMapAttr::get(foldedMap));
2641 return op.getResult();
2646 ? std::min_element(results.begin(), results.end())
2647 : std::max_element(results.begin(), results.end());
2648 if (resultIt == results.end())
2650 return IntegerAttr::get(IndexType::get(op.getContext()), *resultIt);
2654 template <
typename T>
2660 AffineMap oldMap = affineOp.getAffineMap();
2666 if (!llvm::is_contained(newExprs, expr))
2667 newExprs.push_back(expr);
2702 AffineMap oldMap = affineOp.getAffineMap();
2704 affineOp.getMapOperands().take_front(oldMap.
getNumDims());
2706 affineOp.getMapOperands().take_back(oldMap.
getNumSymbols());
2708 auto newDimOperands = llvm::to_vector<8>(dimOperands);
2709 auto newSymOperands = llvm::to_vector<8>(symOperands);
2718 Value symValue = symOperands[symExpr.getPosition()];
2720 producerOps.push_back(producerOp);
2724 Value dimValue = dimOperands[dimExpr.getPosition()];
2726 producerOps.push_back(producerOp);
2733 newExprs.push_back(expr);
2736 if (producerOps.empty())
2743 for (T producerOp : producerOps) {
2744 AffineMap producerMap = producerOp.getAffineMap();
2745 unsigned numProducerDims = producerMap.
getNumDims();
2750 producerOp.getMapOperands().take_front(numProducerDims);
2752 producerOp.getMapOperands().take_back(numProducerSyms);
2753 newDimOperands.append(dimValues.begin(), dimValues.end());
2754 newSymOperands.append(symValues.begin(), symValues.end());
2758 newExprs.push_back(expr.shiftDims(numProducerDims, numUsedDims)
2759 .shiftSymbols(numProducerSyms, numUsedSyms));
2762 numUsedDims += numProducerDims;
2763 numUsedSyms += numProducerSyms;
2769 llvm::to_vector<8>(llvm::concat<Value>(newDimOperands, newSymOperands));
2785 SmallVector<SmallVector<int64_t>> flattenedExprs;
2788 if (!resultExpr.isPureAffine())
2795 if (flattener.operandExprStack.back().size() !=
2799 flattenedExprs.emplace_back(flattener.operandExprStack.back().begin(),
2800 flattener.operandExprStack.back().end());
2804 if (llvm::is_sorted(flattenedExprs))
2808 SmallVector<unsigned> resultPermutation =
2809 llvm::to_vector(llvm::seq<unsigned>(0, map.
getNumResults()));
2810 llvm::sort(resultPermutation, [&](
unsigned lhs,
unsigned rhs) {
2811 return flattenedExprs[lhs] < flattenedExprs[rhs];
2813 SmallVector<AffineExpr> newExprs;
2814 for (
unsigned idx : resultPermutation)
2835 template <
typename T>
2841 AffineMap map = affineOp.getAffineMap();
2850 template <
typename T>
2856 if (affineOp.getMap().getNumResults() != 1)
2859 affineOp.getOperands());
2887 return parseAffineMinMaxOp<AffineMinOp>(parser, result);
2915 return parseAffineMinMaxOp<AffineMaxOp>(parser, result);
2934 IntegerAttr hintInfo;
2936 StringRef readOrWrite, cacheType;
2938 AffineMapAttr mapAttr;
2942 AffinePrefetchOp::getMapAttrStrName(),
2948 AffinePrefetchOp::getLocalityHintAttrStrName(),
2958 if (!readOrWrite.equals(
"read") && !readOrWrite.equals(
"write"))
2960 "rw specifier has to be 'read' or 'write'");
2962 AffinePrefetchOp::getIsWriteAttrStrName(),
2965 if (!cacheType.equals(
"data") && !cacheType.equals(
"instr"))
2967 "cache type has to be 'data' or 'instr'");
2970 AffinePrefetchOp::getIsDataCacheAttrStrName(),
2977 p <<
" " << getMemref() <<
'[';
2978 AffineMapAttr mapAttr =
2979 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName());
2982 p <<
']' <<
", " << (getIsWrite() ?
"write" :
"read") <<
", " 2983 <<
"locality<" << getLocalityHint() <<
">, " 2984 << (getIsDataCache() ?
"data" :
"instr");
2986 (*this)->getAttrs(),
2987 {getMapAttrStrName(), getLocalityHintAttrStrName(),
2988 getIsDataCacheAttrStrName(), getIsWriteAttrStrName()});
2993 auto mapAttr = (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName());
2997 return emitOpError(
"affine.prefetch affine map num results must equal" 3000 return emitOpError(
"too few operands");
3002 if (getNumOperands() != 1)
3003 return emitOpError(
"too few operands");
3007 for (
auto idx : getMapOperands()) {
3009 return emitOpError(
"index must be a dimension or symbol identifier");
3017 results.
add<SimplifyAffineOp<AffinePrefetchOp>>(context);
3035 auto ubs = llvm::to_vector<4>(llvm::map_range(ranges, [&](int64_t value) {
3039 build(builder, result, resultTypes, reductions, lbs, {}, ubs,
3049 assert(llvm::all_of(lbMaps,
3051 return m.
getNumDims() == lbMaps[0].getNumDims() &&
3054 "expected all lower bounds maps to have the same number of dimensions " 3056 assert(llvm::all_of(ubMaps,
3058 return m.
getNumDims() == ubMaps[0].getNumDims() &&
3061 "expected all upper bounds maps to have the same number of dimensions " 3063 assert((lbMaps.empty() || lbMaps[0].getNumInputs() == lbArgs.size()) &&
3064 "expected lower bound maps to have as many inputs as lower bound " 3066 assert((ubMaps.empty() || ubMaps[0].getNumInputs() == ubArgs.size()) &&
3067 "expected upper bound maps to have as many inputs as upper bound " 3074 for (arith::AtomicRMWKind reduction : reductions)
3075 reductionAttrs.push_back(
3087 groups.reserve(groups.size() + maps.size());
3088 exprs.reserve(maps.size());
3093 return AffineMap::get(maps[0].getNumDims(), maps[0].getNumSymbols(), exprs,
3094 maps[0].getContext());
3099 AffineMap lbMap = concatMapsSameInput(lbMaps, lbGroups);
3100 AffineMap ubMap = concatMapsSameInput(ubMaps, ubGroups);
3102 AffineMapAttr::get(lbMap));
3106 AffineMapAttr::get(ubMap));
3115 auto *body =
new Block();
3117 for (
unsigned i = 0, e = steps.size(); i < e; ++i)
3119 bodyRegion->push_back(body);
3120 if (resultTypes.empty())
3121 ensureTerminator(*bodyRegion, builder, result.
location);
3124 Region &AffineParallelOp::getLoopBody() {
return getRegion(); }
3126 unsigned AffineParallelOp::getNumDims() {
return getSteps().size(); }
3128 AffineParallelOp::operand_range AffineParallelOp::getLowerBoundsOperands() {
3129 return getOperands().take_front(getLowerBoundsMap().getNumInputs());
3132 AffineParallelOp::operand_range AffineParallelOp::getUpperBoundsOperands() {
3133 return getOperands().drop_front(getLowerBoundsMap().getNumInputs());
3136 AffineMap AffineParallelOp::getLowerBoundMap(
unsigned pos) {
3137 auto values = getLowerBoundsGroups().getValues<int32_t>();
3139 for (
unsigned i = 0; i < pos; ++i)
3141 return getLowerBoundsMap().getSliceMap(start, values[pos]);
3144 AffineMap AffineParallelOp::getUpperBoundMap(
unsigned pos) {
3145 auto values = getUpperBoundsGroups().getValues<int32_t>();
3147 for (
unsigned i = 0; i < pos; ++i)
3149 return getUpperBoundsMap().getSliceMap(start, values[pos]);
3153 return AffineValueMap(getLowerBoundsMap(), getLowerBoundsOperands());
3157 return AffineValueMap(getUpperBoundsMap(), getUpperBoundsOperands());
3161 if (hasMinMaxBounds())
3170 for (
unsigned i = 0, e = rangesValueMap.
getNumResults(); i < e; ++i) {
3171 auto expr = rangesValueMap.
getResult(i);
3175 out.push_back(cst.getValue());
3180 Block *AffineParallelOp::getBody() {
return &getRegion().
front(); }
3182 OpBuilder AffineParallelOp::getBodyBuilder() {
3183 return OpBuilder(getBody(), std::prev(getBody()->end()));
3188 "operands to map must match number of inputs");
3190 auto ubOperands = getUpperBoundsOperands();
3193 newOperands.append(ubOperands.begin(), ubOperands.end());
3194 (*this)->setOperands(newOperands);
3196 setLowerBoundsMapAttr(AffineMapAttr::get(map));
3201 "operands to map must match number of inputs");
3204 newOperands.append(ubOperands.begin(), ubOperands.end());
3205 (*this)->setOperands(newOperands);
3207 setUpperBoundsMapAttr(AffineMapAttr::get(map));
3210 void AffineParallelOp::setLowerBoundsMap(
AffineMap map) {
3215 setLowerBoundsMapAttr(AffineMapAttr::get(map));
3218 void AffineParallelOp::setUpperBoundsMap(
AffineMap map) {
3223 setUpperBoundsMapAttr(AffineMapAttr::get(map));
3227 setStepsAttr(getBodyBuilder().getI64ArrayAttr(newSteps));
3231 auto numDims = getNumDims();
3234 getSteps().size() != numDims || getBody()->getNumArguments() != numDims) {
3235 return emitOpError() <<
"the number of region arguments (" 3236 << getBody()->getNumArguments()
3237 <<
") and the number of map groups for lower (" 3238 << getLowerBoundsGroups().getNumElements()
3239 <<
") and upper bound (" 3240 << getUpperBoundsGroups().getNumElements()
3241 <<
"), and the number of steps (" << getSteps().size()
3242 <<
") must all match";
3245 unsigned expectedNumLBResults = 0;
3246 for (APInt v : getLowerBoundsGroups())
3247 expectedNumLBResults += v.getZExtValue();
3248 if (expectedNumLBResults != getLowerBoundsMap().getNumResults())
3249 return emitOpError() <<
"expected lower bounds map to have " 3250 << expectedNumLBResults <<
" results";
3251 unsigned expectedNumUBResults = 0;
3252 for (APInt v : getUpperBoundsGroups())
3253 expectedNumUBResults += v.getZExtValue();
3254 if (expectedNumUBResults != getUpperBoundsMap().getNumResults())
3255 return emitOpError() <<
"expected upper bounds map to have " 3256 << expectedNumUBResults <<
" results";
3258 if (getReductions().size() != getNumResults())
3259 return emitOpError(
"a reduction must be specified for each output");
3262 for (
Attribute attr : getReductions()) {
3263 auto intAttr = attr.dyn_cast<IntegerAttr>();
3264 if (!intAttr || !arith::symbolizeAtomicRMWKind(intAttr.getInt()))
3265 return emitOpError(
"invalid reduction attribute");
3271 getLowerBoundsMap().getNumDims())))
3275 getUpperBoundsMap().getNumDims())))
3282 auto newMap = getAffineMap();
3284 if (newMap == getAffineMap() && newOperands == operands)
3286 reset(newMap, newOperands);
3296 bool ubCanonicalized =
succeeded(ub.canonicalize());
3299 if (!lbCanonicalized && !ubCanonicalized)
3302 if (lbCanonicalized)
3304 if (ubCanonicalized)
3305 op.setUpperBounds(ub.getOperands(), ub.getAffineMap());
3322 StringRef keyword) {
3325 ValueRange dimOperands = operands.take_front(numDims);
3326 ValueRange symOperands = operands.drop_front(numDims);
3328 for (llvm::APInt groupSize : group) {
3332 unsigned size = groupSize.getZExtValue();
3337 p << keyword <<
'(';
3347 p <<
" (" << getBody()->getArguments() <<
") = (";
3349 getLowerBoundsOperands(),
"max");
3352 getUpperBoundsOperands(),
"min");
3354 SmallVector<int64_t, 8> steps = getSteps();
3355 bool elideSteps = llvm::all_of(steps, [](int64_t step) {
return step == 1; });
3358 llvm::interleaveComma(steps, p);
3361 if (getNumResults()) {
3363 llvm::interleaveComma(getReductions(), p, [&](
auto &attr) {
3364 arith::AtomicRMWKind sym = *arith::symbolizeAtomicRMWKind(
3365 attr.template cast<IntegerAttr>().getInt());
3366 p <<
"\"" << arith::stringifyAtomicRMWKind(sym) <<
"\"";
3368 p <<
") -> (" << getResultTypes() <<
")";
3375 (*this)->getAttrs(),
3376 {AffineParallelOp::getReductionsAttrStrName(),
3377 AffineParallelOp::getLowerBoundsMapAttrStrName(),
3378 AffineParallelOp::getLowerBoundsGroupsAttrStrName(),
3379 AffineParallelOp::getUpperBoundsMapAttrStrName(),
3380 AffineParallelOp::getUpperBoundsGroupsAttrStrName(),
3381 AffineParallelOp::getStepsAttrStrName()});
3390 ArrayRef<SmallVector<OpAsmParser::UnresolvedOperand>> operands,
3394 "expected operands to be dim or symbol expression");
3397 for (
const auto &
list : operands) {
3398 SmallVector<Value> valueOperands;
3401 for (
Value operand : valueOperands) {
3402 unsigned pos = std::distance(uniqueOperands.begin(),
3403 llvm::find(uniqueOperands, operand));
3404 if (pos == uniqueOperands.size())
3405 uniqueOperands.push_back(operand);
3406 replacements.push_back(
3438 constexpr llvm::StringLiteral tmpAttrStrName =
"__pseudo_bound_map";
3440 StringRef mapName = kind == MinMaxKind::Min
3441 ? AffineParallelOp::getUpperBoundsMapAttrStrName()
3442 : AffineParallelOp::getLowerBoundsMapAttrStrName();
3443 StringRef groupsName =
3444 kind == MinMaxKind::Min
3445 ? AffineParallelOp::getUpperBoundsGroupsAttrStrName()
3446 : AffineParallelOp::getLowerBoundsGroupsAttrStrName();
3458 SmallVector<AffineExpr> flatExprs;
3459 SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> flatDimOperands;
3460 SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> flatSymOperands;
3461 SmallVector<int32_t> numMapsPerGroup;
3462 SmallVector<OpAsmParser::UnresolvedOperand> mapOperands;
3463 auto parseOperands = [&]() {
3465 kind == MinMaxKind::Min ?
"min" :
"max"))) {
3466 mapOperands.clear();
3473 llvm::append_range(flatExprs, map.getValue().getResults());
3474 auto operandsRef = llvm::makeArrayRef(mapOperands);
3475 auto dimsRef = operandsRef.take_front(map.getValue().getNumDims());
3476 SmallVector<OpAsmParser::UnresolvedOperand> dims(dimsRef.begin(),
3478 auto symsRef = operandsRef.drop_front(map.getValue().getNumDims());
3479 SmallVector<OpAsmParser::UnresolvedOperand> syms(symsRef.begin(),
3481 flatDimOperands.append(map.getValue().getNumResults(), dims);
3482 flatSymOperands.append(map.getValue().getNumResults(), syms);
3483 numMapsPerGroup.push_back(map.getValue().getNumResults());
3486 flatSymOperands.emplace_back(),
3487 flatExprs.emplace_back())))
3489 numMapsPerGroup.push_back(1);
3496 unsigned totalNumDims = 0;
3497 unsigned totalNumSyms = 0;
3498 for (
unsigned i = 0, e = flatExprs.size(); i < e; ++i) {
3499 unsigned numDims = flatDimOperands[i].size();
3500 unsigned numSyms = flatSymOperands[i].size();
3501 flatExprs[i] = flatExprs[i]
3502 .shiftDims(numDims, totalNumDims)
3503 .shiftSymbols(numSyms, totalNumSyms);
3504 totalNumDims += numDims;
3505 totalNumSyms += numSyms;
3509 SmallVector<Value> dimOperands, symOperands;
3510 SmallVector<AffineExpr> dimRplacements, symRepacements;
3517 result.
operands.append(dimOperands.begin(), dimOperands.end());
3518 result.
operands.append(symOperands.begin(), symOperands.end());
3521 auto flatMap =
AffineMap::get(totalNumDims, totalNumSyms, flatExprs,
3524 dimRplacements, symRepacements, dimOperands.size(), symOperands.size());
3526 result.
addAttribute(mapName, AffineMapAttr::get(flatMap));
3540 SmallVector<OpAsmParser::Argument, 4> ivs;
3548 AffineMapAttr stepsMapAttr;
3550 SmallVector<OpAsmParser::UnresolvedOperand, 4> stepsMapOperands;
3552 SmallVector<int64_t, 4> steps(ivs.size(), 1);
3553 result.
addAttribute(AffineParallelOp::getStepsAttrStrName(),
3557 AffineParallelOp::getStepsAttrStrName(),
3563 SmallVector<int64_t, 4> steps;
3564 auto stepsMap = stepsMapAttr.getValue();
3565 for (
const auto &result : stepsMap.getResults()) {
3569 "steps must be constant integers");
3570 steps.push_back(constExpr.getValue());
3572 result.
addAttribute(AffineParallelOp::getStepsAttrStrName(),
3578 SmallVector<Attribute, 4> reductions;
3593 arith::symbolizeAtomicRMWKind(attrVal.getValue());
3595 return parser.
emitError(loc,
"invalid reduction value: ") << attrVal;
3597 static_cast<int64_t>(reduction.getValue())));
3604 result.
addAttribute(AffineParallelOp::getReductionsAttrStrName(),
3613 for (
auto &iv : ivs)
3614 iv.type = indexType;
3620 AffineParallelOp::ensureTerminator(*body, builder, result.
location);
3629 auto *parentOp = (*this)->getParentOp();
3630 auto results = parentOp->getResults();
3631 auto operands = getOperands();
3633 if (!isa<AffineParallelOp, AffineIfOp, AffineForOp>(parentOp))
3634 return emitOpError() <<
"only terminates affine.if/for/parallel regions";
3635 if (parentOp->getNumResults() != getNumOperands())
3636 return emitOpError() <<
"parent of yield must have same number of " 3637 "results as the yield operands";
3638 for (
auto it : llvm::zip(results, operands)) {
3639 if (std::get<0>(it).getType() != std::get<1>(it).getType())
3640 return emitOpError() <<
"types mismatch between yield op and its parent";
3653 assert(operands.size() == 1 + map.
getNumInputs() &&
"inconsistent operands");
3656 result.
addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
3657 result.
types.push_back(resultType);
3661 VectorType resultType,
Value memref,
3663 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
3666 result.
addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
3667 result.
types.push_back(resultType);
3671 VectorType resultType,
Value memref,
3673 auto memrefType = memref.
getType().
cast<MemRefType>();
3674 int64_t rank = memrefType.getRank();
3679 build(builder, result, resultType, memref, map, indices);
3682 void AffineVectorLoadOp::getCanonicalizationPatterns(
RewritePatternSet &results,
3684 results.
add<SimplifyAffineOp<AffineVectorLoadOp>>(context);
3692 MemRefType memrefType;
3693 VectorType resultType;
3695 AffineMapAttr mapAttr;
3696 SmallVector<OpAsmParser::UnresolvedOperand, 1> mapOperands;
3700 AffineVectorLoadOp::getMapAttrStrName(),
3711 p <<
" " << getMemRef() <<
'[';
3712 if (AffineMapAttr mapAttr =
3713 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
3717 {getMapAttrStrName()});
3725 if (memrefType.getElementType() != vectorType.getElementType())
3727 "requires memref and vector types of the same elemental type");
3735 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
3736 getMapOperands(), memrefType,
3737 getNumOperands() - 1)))
3753 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
3757 result.
addAttribute(getMapAttrStrName(), AffineMapAttr::get(map));
3764 auto memrefType = memref.
getType().
cast<MemRefType>();
3765 int64_t rank = memrefType.getRank();
3770 build(builder, result, valueToStore, memref, map, indices);
3772 void AffineVectorStoreOp::getCanonicalizationPatterns(
3774 results.
add<SimplifyAffineOp<AffineVectorStoreOp>>(context);
3781 MemRefType memrefType;
3782 VectorType resultType;
3785 AffineMapAttr mapAttr;
3786 SmallVector<OpAsmParser::UnresolvedOperand, 1> mapOperands;
3791 AffineVectorStoreOp::getMapAttrStrName(),
3802 p <<
" " << getValueToStore();
3803 p <<
", " << getMemRef() <<
'[';
3804 if (AffineMapAttr mapAttr =
3805 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
3809 {getMapAttrStrName()});
3810 p <<
" : " <<
getMemRefType() <<
", " << getValueToStore().getType();
3816 *
this, (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
3817 getMapOperands(), memrefType,
3818 getNumOperands() - 2)))
3831 #define GET_OP_CLASSES 3832 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc" static bool isLegalToInline(InlinerInterface &interface, Region *src, Region *insertRegion, bool shouldCloneInlinedRegion, BlockAndValueMapping &valueMapping)
Utility to check that all of the operations within 'src' can be inlined.
operand_range::iterator operand_iterator
TODO: Remove this file when SCCP and integer range analysis have been ported to the new framework...
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
AffineMap getEmptyAffineMap()
Returns a zero result affine map with no dimensions or symbols: () -> ().
static LogicalResult verifyVectorMemoryOp(Operation *op, MemRefType memrefType, VectorType vectorType)
Verify common invariants of affine.vector_load and affine.vector_store.
This class contains a list of basic blocks and a link to the parent operation it is attached to...
virtual ParseResult parseLParen()=0
Parse a ( token.
MLIRContext * getContext() const
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
AffineMap getMultiDimIdentityMap(unsigned rank)
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
Operation is a basic unit of execution within MLIR.
unsigned getNumSymbols() const
unsigned getNumDims() const
static bool isMemRefSizeValidSymbol(AnyMemRefDefOp memrefDefOp, unsigned index, Region *region)
Returns true if the 'index' dimension of the memref defined by memrefDefOp is a statically shaped one...
ParseResult parseDimAndSymbolList(OpAsmParser &parser, SmallVectorImpl< Value > &operands, unsigned &numDims)
Parses dimension and symbol list.
operand_range getOperands()
Returns an iterator on the underlying Value's.
AffineExpr getAffineConstantExpr(int64_t constant, MLIRContext *context)
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
static void printDimAndSymbolList(Operation::operand_iterator begin, Operation::operand_iterator end, unsigned numDims, OpAsmPrinter &printer)
Prints dimension and symbol list.
static Operation * lookupSymbolIn(Operation *op, StringAttr symbol)
Returns the operation registered with the given symbol name with the regions of 'symbolTableOp'.
Block represents an ordered list of Operations.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
static bool hasTrivialZeroTripCount(AffineForOp op)
Returns true if the affine.for has zero iterations in trivial cases.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
virtual void printAffineExprOfSSAIds(AffineExpr expr, ValueRange dimOperands, ValueRange symOperands)=0
Prints an affine expression of SSA ids with SSA id names used instead of dims and symbols...
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
This class represents a single result from folding an operation.
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
unsigned getNumInputs() const
static Operation::operand_range getUpperBoundOperands(AffineForOp forOp)
A trait of region holding operations that defines a new scope for polyhedral optimization purposes...
static LogicalResult foldMemRefCast(Operation *op, Value ignore=nullptr)
This is a common class used for patterns of the form "someop(memrefcast) -> someop".
void push_back(Block *block)
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value...
static bool isValidAffineIndexOperand(Value value, Region *region)
AffineExpr getResult(unsigned i)
static void printAffineMinMaxOp(OpAsmPrinter &p, T op)
static LogicalResult canonicalizeMapExprAndTermOrder(AffineMap &map)
Canonicalize the result expression order of an affine map and return success if the order changed...
RetTy walkPostOrder(AffineExpr expr)
static LogicalResult verifyAffineMinMaxOp(T op)
ParseResult addTypeToList(Type type, SmallVectorImpl< Type > &result)
Add the specified type to the end of the specified type list and return success.
AffineApplyOp makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ValueRange operands)
Returns a composed AffineApplyOp by composing map and operands with other AffineApplyOps supplying th...
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value...
This is the representation of an operand reference.
virtual ParseResult parseArgument(Argument &result, bool allowType=false, bool allowAttrs=false)=0
Parse a single argument with the following syntax:
T lookup(T from) const
Lookup a mapped value within the map.
The OpAsmParser has methods for interacting with the asm parser: parsing things from it...
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
static void printBound(AffineMapAttr boundMap, Operation::operand_range boundOperands, const char *prefix, OpAsmPrinter &p)
virtual ParseResult parseAffineMapOfSSAIds(SmallVectorImpl< UnresolvedOperand > &operands, Attribute &map, StringRef attrName, NamedAttrList &attrs, Delimiter delimiter=Delimiter::Square)=0
Parses an affine map attribute where dims and symbols are SSA operands.
ArrayAttr getI64ArrayAttr(ArrayRef< int64_t > values)
AffineMap shiftDims(unsigned shift, unsigned offset=0) const
Replace dims[offset ...
virtual void startRootUpdate(Operation *op)
This method is used to notify the rewriter that an in-place operation modification is about to happen...
BaseMemRefType getMemRefType(TensorType tensorType, const BufferizationOptions &options, MemRefLayoutAttrInterface layout={}, unsigned memorySpace=0)
Return a MemRefType to which the tensorType can be bufferized.
Attribute erase(StringAttr name)
Erase the attribute with the given name from the list.
ParseResult parseTrailingOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None)
Parse zero or more trailing SSA comma-separated trailing operand references with a specified surround...
static void difference(const AffineValueMap &a, const AffineValueMap &b, AffineValueMap *res)
Return the value map that is the difference of value maps 'a' and 'b', represented as an affine map a...
bool isFunctionOfSymbol(unsigned position) const
Return true if any affine expression involves AffineSymbolExpr position.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
An integer constant appearing in affine expression.
Region * getParent() const
Provide a 'getParent' method for ilist_node_with_parent methods.
virtual ParseResult parseComma()=0
Parse a , token.
void extractForInductionVars(ArrayRef< AffineForOp > forInsts, SmallVectorImpl< Value > *ivs)
Extracts the induction variables from a list of AffineForOps and places them in the output argument i...
static constexpr const bool value
void erase()
Remove this operation from its parent block and delete it.
static void printMinMaxBound(OpAsmPrinter &p, AffineMapAttr mapAttr, DenseIntElementsAttr group, ValueRange operands, StringRef keyword)
Prints a lower(upper) bound of an affine parallel loop with max(min) conditions in it...
SmallVector< Value, 4 > operands
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
LogicalResult verifyInvariantsImpl()
AffineExpr getResult(unsigned idx) const
static AffineMap getConstantMap(int64_t val, MLIRContext *context)
Returns a single constant result affine map.
unsigned getNumInputs() const
Block * getOwner() const
Returns the block that owns this argument.
MutableArrayRef< OpOperand > getOpOperands()
virtual ParseResult parseAffineExprOfSSAIds(SmallVectorImpl< UnresolvedOperand > &dimOperands, SmallVectorImpl< UnresolvedOperand > &symbOperands, AffineExpr &expr)=0
Parses an affine expression where dims and symbols are SSA operands.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
static void canonicalizeMapOrSetAndOperands(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
virtual ParseResult parseArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an arrow followed by a type list.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Merge an affine min/max op to its consumers if its consumer is also an affine min/max op...
This class represents an efficient way to signal success or failure.
AffineMap removeDuplicateExprs(AffineMap map)
Returns a map with the same dimension and symbol count as map, but whose results are the unique affin...
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
AffineMap getSliceMap(unsigned start, unsigned length) const
Returns the map consisting of length expressions starting from start.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
bool isValidDim(Value value)
Returns true if the given Value can be used as a dimension id in the region of the closest surroundin...
static LogicalResult verifyMemoryOpIndexing(Operation *op, AffineMapAttr mapAttr, Operation::operand_range mapOperands, MemRefType memrefType, unsigned numIndexOperands)
Verify common indexing invariants of affine.load, affine.store, affine.vector_load and affine...
virtual void replaceOp(Operation *op, ValueRange newValues)
This method replaces the results of the operation with the specified list of values.
static ParseResult parseBound(bool isLower, OperationState &result, OpAsmParser &p)
Parse a for operation loop bounds.
bool isStrided(MemRefType t)
Return true if the layout for t is compatible with strided semantics.
An attribute that represents a reference to a dense vector or tensor object.
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
virtual ParseResult parseGreater()=0
Parse a '>' token.
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
void addOperands(ValueRange newOperands)
virtual void printAffineMapOfSSAIds(AffineMapAttr mapAttr, ValueRange operands)=0
Prints an affine map of SSA ids, where SSA id names are used in place of dims/symbols.
LogicalResult verifyInvariantsImpl()
IntegerAttr getIntegerAttr(Type type, int64_t value)
void printOptionalArrowTypeList(TypeRange &&types)
Print an optional arrow followed by a type list.
IntegerAttr getI64IntegerAttr(int64_t value)
bool isValidSymbol(Value value)
Returns true if the given value can be used as a symbol in the region of the closest surrounding op t...
static AffineForOp buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds that may or may not be constants.
AffineExpr getAffineSymbolExpr(unsigned position, MLIRContext *context)
Attributes are known-constant values of operations.
ParseResult parseAssignmentList(SmallVectorImpl< Argument > &lhs, SmallVectorImpl< UnresolvedOperand > &rhs)
Parse a list of assignments of the form (x1 = y1, x2 = y2, ...)
void fullyComposeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Given an affine map map and its input operands, this method composes into map, maps of AffineApplyOps...
static void print(spirv::VerCapExtAttr triple, DialectAsmPrinter &printer)
AffineMap replaceDimsAndSymbols(ArrayRef< AffineExpr > dimReplacements, ArrayRef< AffineExpr > symReplacements, unsigned numResultDims, unsigned numResultSyms) const
This method substitutes any uses of dimensions and symbols (e.g.
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
DialectInlinerInterface(Dialect *dialect)
Canonicalize the affine map result expression order of an affine min/max operation.
static bool isDimOpValidSymbol(OpTy dimOp, Region *region)
Returns true if the result of the dim op is a valid symbol for region.
void print(OpAsmPrinter &p)
IntegerType getIntegerType(unsigned width)
virtual ParseResult parseRParen()=0
Parse a ) token.
This is the interface that must be implemented by the dialects of operations to be inlined...
Base type for affine expression.
void canonicalizeMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Modifies both map and operands in-place so as to:
A trait used to provide symbol table functionalities to a region operation.
void print(OpAsmPrinter &p)
LogicalResult constantFold(ArrayRef< Attribute > operandConstants, SmallVectorImpl< Attribute > &results) const
Folds the results of the application of an affine map on the provided operands to a constant if possi...
void canonicalizeSetAndOperands(IntegerSet *set, SmallVectorImpl< Value > *operands)
Canonicalizes an integer set the same way canonicalizeMapAndOperands does for affine maps...
This class provides an abstraction over the various different ranges of value types.
static void build(OpBuilder &builder, OperationState &result, Value tagMemRef, AffineMap tagMap, ValueRange tagIndices, Value numElements)
static ParseResult deduplicateAndResolveOperands(OpAsmParser &parser, ArrayRef< SmallVector< OpAsmParser::UnresolvedOperand >> operands, SmallVectorImpl< Value > &uniqueOperands, SmallVectorImpl< AffineExpr > &replacements, AffineExprKind kind)
Given a list of lists of parsed operands, populates uniqueOperands with unique operands.
void addTypes(ArrayRef< Type > newTypes)
static void canonicalizePromotedSymbols(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
ParseResult parseKeyword(StringRef keyword, const Twine &msg="")
Parse a given keyword.
virtual ParseResult parseLess()=0
Parse a '<' token.
SmallVector< Value, 4 > applyMapToValues(OpBuilder &b, Location loc, AffineMap map, ValueRange values)
Returns the values obtained by applying map to the list of values.
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
unsigned getNumResults() const
This represents an operation in an abstracted form, suitable for use with the builder APIs...
void mergeBlockBefore(Block *source, Operation *op, ValueRange argValues=llvm::None)
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true...
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued...
bool isForInductionVar(Value val)
Returns true if the provided value is the induction variable of a AffineForOp.
AffineDmaWaitOp blocks until the completion of a DMA operation associated with the tag element 'tag[i...
DenseIntElementsAttr getI32TensorAttr(ArrayRef< int32_t > values)
Tensor-typed DenseIntElementsAttr getters.
Parens surrounding zero or more operands.
static Operation::operand_range getLowerBoundOperands(AffineForOp forOp)
unsigned getNumDims() const
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
AffineMap getAffineMap() const
BlockArgListType getArguments()
AffineBound represents a lower or upper bound in the for operation.
This class represents an argument of a Block.
ParseResult resolveOperands(ArrayRef< UnresolvedOperand > operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
static void build(OpBuilder &builder, OperationState &result, Value srcMemRef, AffineMap srcMap, ValueRange srcIndices, Value destMemRef, AffineMap dstMap, ValueRange destIndices, Value tagMemRef, AffineMap tagMap, ValueRange tagIndices, Value numElements, Value stride=nullptr, Value elementsPerStride=nullptr)
ArrayRef< AffineExpr > getResults() const
AffineMap replace(AffineExpr expr, AffineExpr replacement, unsigned numResultDims, unsigned numResultSyms) const
Sparse replace method.
static LogicalResult canonicalizeLoopBounds(AffineForOp forOp)
Canonicalize the bounds of the given loop.
AffineExpr getAffineDimExpr(unsigned position, MLIRContext *context)
These free functions allow clients of the API to not use classes in detail.
static ParseResult parse(OpAsmParser &parser, OperationState &result)
virtual void finalizeRootUpdate(Operation *op)
This method is used to signal the end of a root update on the given operation.
Remove duplicated expressions in affine min/max ops.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
static void buildAffineLoopNestImpl(OpBuilder &builder, Location loc, BoundListTy lbs, BoundListTy ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn, LoopCreatorTy &&loopCreatorFn)
Builds an affine loop nest, using "loopCreatorFn" to create individual loop operations.
Operation * getParentOp()
Return the parent operation this region is attached to.
virtual ParseResult parseOptionalRParen()=0
Parse a ) token if present.
AffineForOp getForInductionVarOwner(Value val)
Returns the loop parent of an induction variable.
static ParseResult parse(OpAsmParser &parser, OperationState &result)
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
An attribute that represents a reference to a splat vector or tensor constant, meaning all of the ele...
static LogicalResult verifyDimAndSymbolIdentifiers(OpTy &op, Operation::operand_range operands, unsigned numDims)
Utility function to verify that a set of operands are valid dimension and symbol identifiers.
static LogicalResult foldLoopBounds(AffineForOp forOp)
Fold the constant bounds of a loop.
bool isTopLevelValue(Value value)
TODO: These should be renamed if they are on the mlir namespace.
Operation * getTerminator()
Get the terminator operation of this block.
static Value createFoldedComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ValueRange operandsRef)
Fully compose map with operands and canonicalize the result.
virtual void printRegionArgument(BlockArgument arg, ArrayRef< NamedAttribute > argAttrs={}, bool omitType=false)=0
Print a block argument in the usual format of: ssaName : type {attr1=42} loc("here") where location p...
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
void pop_back()
Pop last element from list.
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values...
RAII guard to reset the insertion point of the builder when destroyed.
LogicalResult fold(ArrayRef< Attribute > cstOperands, SmallVectorImpl< OpFoldResult > &results)
This class represents a successor of a region.
Region * addRegion()
Create a region that should be attached to the operation.
Region * getAffineScope(Operation *op)
Returns the closest region enclosing op that is held by an operation with trait AffineScope; nullptr ...
This class is a general helper class for creating context-global objects like types, attributes, and affine expressions.
AffineMap simplifyAffineMap(AffineMap map)
Simplifies an affine map by simplifying its underlying AffineExpr results.
Type getType() const
Return the type of this value.
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&... args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments...
static ParseResult parseAffineMinMaxOp(OpAsmParser &parser, OperationState &result)
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replaces the result op with a new op that is created without verification.
This class provides the API for ops that are known to be isolated from above.
AffineMap getDimIdentityMap()
virtual ParseResult parseArgumentList(SmallVectorImpl< Argument > &result, Delimiter delimiter=Delimiter::None, bool allowType=false, bool allowAttrs=false)=0
Parse zero or more arguments with a specified surrounding delimiter.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
static int64_t getNumElements(ShapedType type)
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
A dimensional identifier appearing in an affine expression.
Specialization of arith.constant op that returns an integer of index type.
static VectorType vectorType(CodeGen &codegen, Type etp)
Constructs vector type.
virtual ParseResult parseRegion(Region ®ion, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region.
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
bool isFunctionOfDim(unsigned position) const
Return true if any affine expression involves AffineDimExpr position.
BoolAttr getBoolAttr(bool value)
virtual ParseResult parseType(Type &result)=0
Parse a type.
static LogicalResult replaceDimOrSym(AffineMap *map, unsigned dimOrSymbolPosition, SmallVectorImpl< Value > &dims, SmallVectorImpl< Value > &syms)
Replace all occurrences of AffineExpr at position pos in map by the defining AffineApplyOp expression...
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
MLIRContext is the top-level object for a collection of MLIR operations.
This class represents an operand of an operation.
AffineForOp replaceForOpWithNewYields(OpBuilder &b, AffineForOp loop, ValueRange newIterOperands, ValueRange newYieldedValues, ValueRange newIterArgs, bool replaceLoopResults=true)
Replace loop with a new loop where newIterOperands are appended with new initialization values and ne...
AffineDmaStartOp starts a non-blocking DMA operation that transfers data from a source memref to a de...
This class implements the operand iterators for the Operation class.
Region * getParentRegion()
Returns the region to which the instruction belongs.
static ParseResult parseAffineMapWithMinMax(OpAsmParser &parser, OperationState &result, MinMaxKind kind)
Parses an affine map that can contain a min/max for groups of its results, e.g., max(expr-1, expr-2), expr-3, max(expr-4, expr-5, expr-6).
LogicalResult verify(Operation *op, bool verifyRecursively=true)
Perform (potentially expensive) checks of invariants, used to detect compiler bugs, on this operation and any nested operations.
AffineMap getConstantAffineMap(int64_t val)
Returns a single constant result affine map with 0 dimensions and 0 symbols.
static VectorType getVectorType(Type scalarTy, const VectorizationStrategy *strategy)
Returns the vector type resulting from applying the provided vectorization strategy on the scalar typ...
An AffineValueMap is an affine map plus its ML value operands and results for analysis purposes...
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseEqual()=0
Parse a = token.
MLIRContext * getContext() const
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
SmallVector< std::unique_ptr< Region >, 1 > regions
Regions that the op will hold.
LogicalResult canonicalize()
Attempts to canonicalize the map and operands.
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers...
static SmallVector< AffineMap, 4 > inferFromExprList(ArrayRef< ArrayRef< AffineExpr >> exprsList)
Returns a vector of AffineMaps; each with as many results as exprs.size(), as many dims as the larges...
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
Square brackets supporting zero or more ops, or nothing.
MLIRContext * getContext() const
This class represents success/failure for parsing-like operations that find it important to chain tog...
static bool remainsLegalAfterInline(Value value, Region *src, Region *dest, const BlockAndValueMapping &mapping, function_ref< bool(Value, Region *)> legalityCheck)
Checks if value known to be a legal affine dimension or symbol in src region remains legal if the ope...
ArrayRef< Value > getOperands() const
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
This class helps build Operations.
ArrayAttr getArrayAttr(ArrayRef< Attribute > value)
std::enable_if<!std::is_base_of< Attribute, T >::value||std::is_same< Attribute, T >::value, T >::type getSplatValue() const
Return the splat value for this attribute.
This class provides an abstraction over the different types of ranges over Values.
static void composeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Iterate over operands and fold away all those produced by an AffineApplyOp iteratively.
virtual ParseResult parseOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None, bool allowResultNumber=true, int requiredOperandCount=-1)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter...
virtual ParseResult parseOptionalArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an optional arrow followed by a type list.
bool isSingleConstant() const
Returns true if this affine map is a single result constant function.
static AffineForOp buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb, int64_t ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds known to be constants.
virtual ParseResult parseColonType(Type &result)=0
Parse a colon followed by a type.
static Operation * materializeConstant(Dialect *dialect, OpBuilder &builder, Attribute value, Type type, Location loc)
A utility function used to materialize a constant for a given attribute and type. ...
LogicalResult fold(ArrayRef< Attribute > cstOperands, SmallVectorImpl< OpFoldResult > &results)
static OpFoldResult foldMinMaxOp(T op, ArrayRef< Attribute > operands)
Fold an affine min or max operation with the given operands.
An attribute that represents a reference to a dense integer vector or tensor object.
A symbolic identifier appearing in an affine expression.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
SmallVector< Type, 4 > types
Types of the results of this operation.
virtual void eraseBlock(Block *block)
This method erases all operations in a block.
void buildAffineLoopNest(OpBuilder &builder, Location loc, ArrayRef< int64_t > lbs, ArrayRef< int64_t > ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn=nullptr)
Builds a perfect nest of affine.for loops, i.e., each loop except the innermost one contains only ano...
unsigned getNumResults() const
An integer set representing a conjunction of one or more affine equalities and inequalities.