22 #include "llvm/ADT/ScopeExit.h"
23 #include "llvm/ADT/SmallBitVector.h"
24 #include "llvm/ADT/SmallVectorExtras.h"
25 #include "llvm/ADT/TypeSwitch.h"
26 #include "llvm/Support/Debug.h"
33 #define DEBUG_TYPE "affine-ops"
35 #include "mlir/Dialect/Affine/IR/AffineOpsDialect.cpp.inc"
42 if (
auto arg = llvm::dyn_cast<BlockArgument>(value))
43 return arg.getParentRegion() == region;
66 if (llvm::isa<BlockArgument>(value))
67 return legalityCheck(mapping.
lookup(value), dest);
74 bool isDimLikeOp = isa<ShapedDimOpInterface>(value.
getDefiningOp());
85 return llvm::all_of(values, [&](
Value v) {
92 template <
typename OpTy>
95 static_assert(llvm::is_one_of<OpTy, AffineReadOpInterface,
96 AffineWriteOpInterface>::value,
97 "only ops with affine read/write interface are supported");
104 dimOperands, src, dest, mapping,
108 symbolOperands, src, dest, mapping,
125 op.getMapOperands(), src, dest, mapping,
130 op.getMapOperands(), src, dest, mapping,
157 if (!isa<AffineParallelOp, AffineForOp, AffineIfOp>(destOp))
162 if (!llvm::hasSingleElement(*src))
170 if (
auto iface = dyn_cast<MemoryEffectOpInterface>(op)) {
171 if (iface.hasNoEffect())
179 .Case<AffineApplyOp, AffineReadOpInterface,
180 AffineWriteOpInterface>([&](
auto op) {
205 isa<AffineForOp, AffineParallelOp, AffineIfOp>(parentOp);
209 bool shouldAnalyzeRecursively(
Operation *op)
const final {
return true; }
217 void AffineDialect::initialize() {
220 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc"
222 addInterfaces<AffineInlinerInterface>();
230 if (
auto poison = dyn_cast<ub::PoisonAttr>(value))
231 return builder.
create<ub::PoisonOp>(loc, type, poison);
232 return arith::ConstantOp::materialize(builder, value, type, loc);
240 if (
auto arg = llvm::dyn_cast<BlockArgument>(value)) {
256 while (
auto *parentOp = curOp->getParentOp()) {
279 auto *parentOp = llvm::cast<BlockArgument>(value).getOwner()->getParentOp();
281 isa<AffineForOp, AffineParallelOp>(parentOp));
302 auto *parentOp = llvm::cast<BlockArgument>(value).getOwner()->
getParentOp();
303 return isa<AffineForOp, AffineParallelOp>(parentOp);
307 if (
auto applyOp = dyn_cast<AffineApplyOp>(op))
308 return applyOp.isValidDim(region);
311 if (
auto dimOp = dyn_cast<ShapedDimOpInterface>(op))
319 template <
typename AnyMemRefDefOp>
322 MemRefType memRefType = memrefDefOp.getType();
325 if (index >= memRefType.getRank()) {
330 if (!memRefType.isDynamicDim(index))
333 unsigned dynamicDimPos = memRefType.getDynamicDimIndex(index);
334 return isValidSymbol(*(memrefDefOp.getDynamicSizes().begin() + dynamicDimPos),
346 if (llvm::isa<BlockArgument>(dimOp.getShapedValue()))
354 if (!index.has_value())
357 int64_t i = index.value();
359 .Case<memref::ViewOp, memref::SubViewOp, memref::AllocOp>(
361 .Default([](
Operation *) {
return false; });
427 if (
auto applyOp = dyn_cast<AffineApplyOp>(defOp))
428 return applyOp.isValidSymbol(region);
431 if (
auto dimOp = dyn_cast<ShapedDimOpInterface>(defOp))
455 printer <<
'(' << operands.take_front(numDims) <<
')';
456 if (operands.size() > numDims)
457 printer <<
'[' << operands.drop_front(numDims) <<
']';
467 numDims = opInfos.size();
481 template <
typename OpTy>
486 for (
auto operand : operands) {
487 if (opIt++ < numDims) {
489 return op.
emitOpError(
"operand cannot be used as a dimension id");
491 return op.
emitOpError(
"operand cannot be used as a symbol");
502 return AffineValueMap(getAffineMap(), getOperands(), getResult());
509 AffineMapAttr mapAttr;
515 auto map = mapAttr.getValue();
517 if (map.getNumDims() != numDims ||
518 numDims + map.getNumSymbols() != result.
operands.size()) {
520 "dimension or symbol index mismatch");
523 result.
types.append(map.getNumResults(), indexTy);
528 p <<
" " << getMapAttr();
530 getAffineMap().getNumDims(), p);
541 "operand count and affine map dimension and symbol count must match");
545 return emitOpError(
"mapping must produce one value");
553 return llvm::all_of(getOperands(),
561 return llvm::all_of(getOperands(),
568 return llvm::all_of(getOperands(),
575 return llvm::all_of(getOperands(), [&](
Value operand) {
581 auto map = getAffineMap();
584 auto expr = map.getResult(0);
585 if (
auto dim = dyn_cast<AffineDimExpr>(expr))
586 return getOperand(dim.getPosition());
587 if (
auto sym = dyn_cast<AffineSymbolExpr>(expr))
588 return getOperand(map.getNumDims() + sym.getPosition());
592 bool hasPoison =
false;
594 map.constantFold(adaptor.getMapOperands(), result, &hasPoison);
614 auto dimExpr = dyn_cast<AffineDimExpr>(e);
624 Value operand = operands[dimExpr.getPosition()];
625 int64_t operandDivisor = 1;
629 if (forOp.hasConstantLowerBound() && forOp.getConstantLowerBound() == 0) {
630 operandDivisor = forOp.getStepAsInt();
632 uint64_t lbLargestKnownDivisor =
633 forOp.getLowerBoundMap().getLargestKnownDivisorOfMapExprs();
634 operandDivisor =
std::gcd(lbLargestKnownDivisor, forOp.getStepAsInt());
637 return operandDivisor;
644 if (
auto constExpr = dyn_cast<AffineConstantExpr>(e)) {
645 int64_t constVal = constExpr.getValue();
646 return constVal >= 0 && constVal < k;
648 auto dimExpr = dyn_cast<AffineDimExpr>(e);
651 Value operand = operands[dimExpr.getPosition()];
655 if (forOp.hasConstantLowerBound() && forOp.getConstantLowerBound() >= 0 &&
656 forOp.hasConstantUpperBound() && forOp.getConstantUpperBound() <= k) {
672 auto bin = dyn_cast<AffineBinaryOpExpr>(e);
680 quotientTimesDiv = llhs;
686 quotientTimesDiv = rlhs;
696 if (forOp && forOp.hasConstantLowerBound())
697 return forOp.getConstantLowerBound();
704 if (!forOp || !forOp.hasConstantUpperBound())
709 if (forOp.hasConstantLowerBound()) {
710 return forOp.getConstantUpperBound() - 1 -
711 (forOp.getConstantUpperBound() - forOp.getConstantLowerBound() - 1) %
712 forOp.getStepAsInt();
714 return forOp.getConstantUpperBound() - 1;
725 constLowerBounds.reserve(operands.size());
726 constUpperBounds.reserve(operands.size());
727 for (
Value operand : operands) {
732 if (
auto constExpr = dyn_cast<AffineConstantExpr>(expr))
733 return constExpr.getValue();
748 constLowerBounds.reserve(operands.size());
749 constUpperBounds.reserve(operands.size());
750 for (
Value operand : operands) {
755 std::optional<int64_t> lowerBound;
756 if (
auto constExpr = dyn_cast<AffineConstantExpr>(expr)) {
757 lowerBound = constExpr.getValue();
760 constLowerBounds, constUpperBounds,
771 auto binExpr = dyn_cast<AffineBinaryOpExpr>(expr);
782 binExpr = dyn_cast<AffineBinaryOpExpr>(expr);
790 lhs = binExpr.getLHS();
791 rhs = binExpr.getRHS();
792 auto rhsConst = dyn_cast<AffineConstantExpr>(rhs);
796 int64_t rhsConstVal = rhsConst.getValue();
798 if (rhsConstVal <= 0)
803 std::optional<int64_t> lhsLbConst =
805 std::optional<int64_t> lhsUbConst =
807 if (lhsLbConst && lhsUbConst) {
808 int64_t lhsLbConstVal = *lhsLbConst;
809 int64_t lhsUbConstVal = *lhsUbConst;
813 floorDiv(lhsLbConstVal, rhsConstVal) ==
814 floorDiv(lhsUbConstVal, rhsConstVal)) {
822 ceilDiv(lhsLbConstVal, rhsConstVal) ==
823 ceilDiv(lhsUbConstVal, rhsConstVal)) {
830 lhsLbConstVal < rhsConstVal && lhsUbConstVal < rhsConstVal) {
842 if (
isQTimesDPlusR(lhs, operands, divisor, quotientTimesDiv, rem)) {
843 if (rhsConstVal % divisor == 0 &&
845 expr = quotientTimesDiv.
floorDiv(rhsConst);
846 }
else if (divisor % rhsConstVal == 0 &&
848 expr = rem % rhsConst;
874 if (operands.empty())
880 constLowerBounds.reserve(operands.size());
881 constUpperBounds.reserve(operands.size());
882 for (
Value operand : operands) {
896 if (
auto constExpr = dyn_cast<AffineConstantExpr>(e)) {
897 lowerBounds.push_back(constExpr.getValue());
898 upperBounds.push_back(constExpr.getValue());
900 lowerBounds.push_back(
902 constLowerBounds, constUpperBounds,
904 upperBounds.push_back(
906 constLowerBounds, constUpperBounds,
915 unsigned i = exprEn.index();
917 if (lowerBounds[i] && upperBounds[i] && *lowerBounds[i] == *upperBounds[i])
922 if (!upperBounds[i]) {
923 irredundantExprs.push_back(e);
929 auto otherLowerBound = en.value();
930 unsigned pos = en.index();
931 if (pos == i || !otherLowerBound)
933 if (*otherLowerBound > *upperBounds[i])
935 if (*otherLowerBound < *upperBounds[i])
940 if (upperBounds[pos] && lowerBounds[i] &&
941 lowerBounds[i] == upperBounds[i] &&
942 otherLowerBound == *upperBounds[pos] && i < pos)
946 irredundantExprs.push_back(e);
948 if (!lowerBounds[i]) {
949 irredundantExprs.push_back(e);
954 auto otherUpperBound = en.value();
955 unsigned pos = en.index();
956 if (pos == i || !otherUpperBound)
958 if (*otherUpperBound < *lowerBounds[i])
960 if (*otherUpperBound > *lowerBounds[i])
962 if (lowerBounds[pos] && upperBounds[i] &&
963 lowerBounds[i] == upperBounds[i] &&
964 otherUpperBound == lowerBounds[pos] && i < pos)
968 irredundantExprs.push_back(e);
980 static void LLVM_ATTRIBUTE_UNUSED
982 assert(map.
getNumInputs() == operands.size() &&
"invalid operands for map");
988 newResults.push_back(expr);
1005 unsigned dimOrSymbolPosition,
1009 bool isDimReplacement = (dimOrSymbolPosition < dims.size());
1010 unsigned pos = isDimReplacement ? dimOrSymbolPosition
1011 : dimOrSymbolPosition - dims.size();
1012 Value &v = isDimReplacement ? dims[pos] : syms[pos];
1025 AffineMap composeMap = affineApply.getAffineMap();
1026 assert(composeMap.
getNumResults() == 1 &&
"affine.apply with >1 results");
1028 affineApply.getMapOperands().end());
1042 dims.append(composeDims.begin(), composeDims.end());
1043 syms.append(composeSyms.begin(), composeSyms.end());
1044 *map = map->
replace(toReplace, replacementExpr, dims.size(), syms.size());
1072 bool changed =
false;
1073 for (
unsigned pos = 0; pos != dims.size() + syms.size(); ++pos)
1085 unsigned nDims = 0, nSyms = 0;
1087 dimReplacements.reserve(dims.size());
1088 symReplacements.reserve(syms.size());
1089 for (
auto *container : {&dims, &syms}) {
1090 bool isDim = (container == &dims);
1091 auto &repls = isDim ? dimReplacements : symReplacements;
1093 Value v = en.value();
1097 "map is function of unexpected expr@pos");
1103 operands->push_back(v);
1116 while (llvm::any_of(*operands, [](
Value v) {
1130 return b.
create<AffineApplyOp>(loc, map, valueOperands);
1150 for (
unsigned i : llvm::seq<unsigned>(0, map.
getNumResults())) {
1157 llvm::append_range(dims,
1159 llvm::append_range(symbols,
1166 operands = llvm::to_vector(llvm::concat<Value>(dims, symbols));
1175 assert(map.
getNumResults() == 1 &&
"building affine.apply with !=1 result");
1185 AffineApplyOp applyOp =
1190 for (
unsigned i = 0, e = constOperands.size(); i != e; ++i)
1195 if (
failed(applyOp->fold(constOperands, foldResults)) ||
1196 foldResults.empty()) {
1198 listener->notifyOperationInserted(applyOp);
1199 return applyOp.getResult();
1203 assert(foldResults.size() == 1 &&
"expected 1 folded result");
1204 return foldResults.front();
1220 return llvm::map_to_vector(llvm::seq<unsigned>(0, map.
getNumResults()),
1222 return makeComposedFoldedAffineApply(
1223 b, loc, map.getSubMap({i}), operands);
1227 template <
typename OpTy>
1239 return makeComposedMinMax<AffineMinOp>(b, loc, map, operands);
1242 template <
typename OpTy>
1254 auto minMaxOp = makeComposedMinMax<OpTy>(newBuilder, loc, map, operands);
1258 for (
unsigned i = 0, e = constOperands.size(); i != e; ++i)
1263 if (
failed(minMaxOp->fold(constOperands, foldResults)) ||
1264 foldResults.empty()) {
1266 listener->notifyOperationInserted(minMaxOp);
1267 return minMaxOp.getResult();
1271 assert(foldResults.size() == 1 &&
"expected 1 folded result");
1272 return foldResults.front();
1279 return makeComposedFoldedMinMax<AffineMinOp>(b, loc, map, operands);
1286 return makeComposedFoldedMinMax<AffineMaxOp>(b, loc, map, operands);
1291 template <
class MapOrSet>
1294 if (!mapOrSet || operands->empty())
1297 assert(mapOrSet->getNumInputs() == operands->size() &&
1298 "map/set inputs must match number of operands");
1300 auto *context = mapOrSet->getContext();
1302 resultOperands.reserve(operands->size());
1304 remappedSymbols.reserve(operands->size());
1305 unsigned nextDim = 0;
1306 unsigned nextSym = 0;
1307 unsigned oldNumSyms = mapOrSet->getNumSymbols();
1309 for (
unsigned i = 0, e = mapOrSet->getNumInputs(); i != e; ++i) {
1310 if (i < mapOrSet->getNumDims()) {
1314 remappedSymbols.push_back((*operands)[i]);
1317 resultOperands.push_back((*operands)[i]);
1320 resultOperands.push_back((*operands)[i]);
1324 resultOperands.append(remappedSymbols.begin(), remappedSymbols.end());
1325 *operands = resultOperands;
1326 *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, {}, nextDim,
1327 oldNumSyms + nextSym);
1329 assert(mapOrSet->getNumInputs() == operands->size() &&
1330 "map/set inputs must match number of operands");
1334 template <
class MapOrSet>
1337 static_assert(llvm::is_one_of<MapOrSet, AffineMap, IntegerSet>::value,
1338 "Argument must be either of AffineMap or IntegerSet type");
1340 if (!mapOrSet || operands->empty())
1343 assert(mapOrSet->getNumInputs() == operands->size() &&
1344 "map/set inputs must match number of operands");
1346 canonicalizePromotedSymbols<MapOrSet>(mapOrSet, operands);
1349 llvm::SmallBitVector usedDims(mapOrSet->getNumDims());
1350 llvm::SmallBitVector usedSyms(mapOrSet->getNumSymbols());
1352 if (
auto dimExpr = dyn_cast<AffineDimExpr>(expr))
1353 usedDims[dimExpr.getPosition()] =
true;
1354 else if (
auto symExpr = dyn_cast<AffineSymbolExpr>(expr))
1355 usedSyms[symExpr.getPosition()] =
true;
1358 auto *context = mapOrSet->getContext();
1361 resultOperands.reserve(operands->size());
1363 llvm::SmallDenseMap<Value, AffineExpr, 8> seenDims;
1365 unsigned nextDim = 0;
1366 for (
unsigned i = 0, e = mapOrSet->getNumDims(); i != e; ++i) {
1369 auto it = seenDims.find((*operands)[i]);
1370 if (it == seenDims.end()) {
1372 resultOperands.push_back((*operands)[i]);
1373 seenDims.insert(std::make_pair((*operands)[i], dimRemapping[i]));
1375 dimRemapping[i] = it->second;
1379 llvm::SmallDenseMap<Value, AffineExpr, 8> seenSymbols;
1381 unsigned nextSym = 0;
1382 for (
unsigned i = 0, e = mapOrSet->getNumSymbols(); i != e; ++i) {
1388 IntegerAttr operandCst;
1389 if (
matchPattern((*operands)[i + mapOrSet->getNumDims()],
1396 auto it = seenSymbols.find((*operands)[i + mapOrSet->getNumDims()]);
1397 if (it == seenSymbols.end()) {
1399 resultOperands.push_back((*operands)[i + mapOrSet->getNumDims()]);
1400 seenSymbols.insert(std::make_pair((*operands)[i + mapOrSet->getNumDims()],
1403 symRemapping[i] = it->second;
1406 *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, symRemapping,
1408 *operands = resultOperands;
1413 canonicalizeMapOrSetAndOperands<AffineMap>(map, operands);
1418 canonicalizeMapOrSetAndOperands<IntegerSet>(set, operands);
1425 template <
typename AffineOpTy>
1437 llvm::is_one_of<AffineOpTy, AffineLoadOp, AffinePrefetchOp,
1438 AffineStoreOp, AffineApplyOp, AffineMinOp, AffineMaxOp,
1439 AffineVectorStoreOp, AffineVectorLoadOp>::value,
1440 "affine load/store/vectorstore/vectorload/apply/prefetch/min/max op "
1442 auto map = affineOp.getAffineMap();
1444 auto oldOperands = affineOp.getMapOperands();
1449 if (map == oldMap && std::equal(oldOperands.begin(), oldOperands.end(),
1450 resultOperands.begin()))
1453 replaceAffineOp(rewriter, affineOp, map, resultOperands);
1461 void SimplifyAffineOp<AffineLoadOp>::replaceAffineOp(
1468 void SimplifyAffineOp<AffinePrefetchOp>::replaceAffineOp(
1472 prefetch, prefetch.getMemref(), map, mapOperands,
1473 prefetch.getLocalityHint(), prefetch.getIsWrite(),
1474 prefetch.getIsDataCache());
1477 void SimplifyAffineOp<AffineStoreOp>::replaceAffineOp(
1481 store, store.getValueToStore(), store.getMemRef(), map, mapOperands);
1484 void SimplifyAffineOp<AffineVectorLoadOp>::replaceAffineOp(
1488 vectorload, vectorload.getVectorType(), vectorload.getMemRef(), map,
1492 void SimplifyAffineOp<AffineVectorStoreOp>::replaceAffineOp(
1496 vectorstore, vectorstore.getValueToStore(), vectorstore.getMemRef(), map,
1501 template <
typename AffineOpTy>
1502 void SimplifyAffineOp<AffineOpTy>::replaceAffineOp(
1511 results.
add<SimplifyAffineOp<AffineApplyOp>>(context);
1542 p <<
" " << getSrcMemRef() <<
'[';
1544 p <<
"], " << getDstMemRef() <<
'[';
1546 p <<
"], " << getTagMemRef() <<
'[';
1550 p <<
", " << getStride();
1551 p <<
", " << getNumElementsPerStride();
1553 p <<
" : " << getSrcMemRefType() <<
", " << getDstMemRefType() <<
", "
1554 << getTagMemRefType();
1566 AffineMapAttr srcMapAttr;
1569 AffineMapAttr dstMapAttr;
1572 AffineMapAttr tagMapAttr;
1587 getSrcMapAttrStrName(),
1591 getDstMapAttrStrName(),
1595 getTagMapAttrStrName(),
1604 if (!strideInfo.empty() && strideInfo.size() != 2) {
1606 "expected two stride related operands");
1608 bool isStrided = strideInfo.size() == 2;
1613 if (types.size() != 3)
1631 if (srcMapOperands.size() != srcMapAttr.getValue().getNumInputs() ||
1632 dstMapOperands.size() != dstMapAttr.getValue().getNumInputs() ||
1633 tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1635 "memref operand count not equal to map.numInputs");
1640 if (!llvm::isa<MemRefType>(getOperand(getSrcMemRefOperandIndex()).getType()))
1641 return emitOpError(
"expected DMA source to be of memref type");
1642 if (!llvm::isa<MemRefType>(getOperand(getDstMemRefOperandIndex()).getType()))
1643 return emitOpError(
"expected DMA destination to be of memref type");
1644 if (!llvm::isa<MemRefType>(getOperand(getTagMemRefOperandIndex()).getType()))
1645 return emitOpError(
"expected DMA tag to be of memref type");
1647 unsigned numInputsAllMaps = getSrcMap().getNumInputs() +
1648 getDstMap().getNumInputs() +
1649 getTagMap().getNumInputs();
1650 if (getNumOperands() != numInputsAllMaps + 3 + 1 &&
1651 getNumOperands() != numInputsAllMaps + 3 + 1 + 2) {
1652 return emitOpError(
"incorrect number of operands");
1656 for (
auto idx : getSrcIndices()) {
1657 if (!idx.getType().isIndex())
1658 return emitOpError(
"src index to dma_start must have 'index' type");
1661 "src index must be a valid dimension or symbol identifier");
1663 for (
auto idx : getDstIndices()) {
1664 if (!idx.getType().isIndex())
1665 return emitOpError(
"dst index to dma_start must have 'index' type");
1668 "dst index must be a valid dimension or symbol identifier");
1670 for (
auto idx : getTagIndices()) {
1671 if (!idx.getType().isIndex())
1672 return emitOpError(
"tag index to dma_start must have 'index' type");
1675 "tag index must be a valid dimension or symbol identifier");
1686 void AffineDmaStartOp::getEffects(
1712 p <<
" " << getTagMemRef() <<
'[';
1717 p <<
" : " << getTagMemRef().getType();
1728 AffineMapAttr tagMapAttr;
1737 getTagMapAttrStrName(),
1746 if (!llvm::isa<MemRefType>(type))
1748 "expected tag to be of memref type");
1750 if (tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1752 "tag memref operand count != to map.numInputs");
1757 if (!llvm::isa<MemRefType>(getOperand(0).getType()))
1758 return emitOpError(
"expected DMA tag to be of memref type");
1760 for (
auto idx : getTagIndices()) {
1761 if (!idx.getType().isIndex())
1762 return emitOpError(
"index to dma_wait must have 'index' type");
1765 "index must be a valid dimension or symbol identifier");
1776 void AffineDmaWaitOp::getEffects(
1792 ValueRange iterArgs, BodyBuilderFn bodyBuilder) {
1793 assert(((!lbMap && lbOperands.empty()) ||
1795 "lower bound operand count does not match the affine map");
1796 assert(((!ubMap && ubOperands.empty()) ||
1798 "upper bound operand count does not match the affine map");
1799 assert(step > 0 &&
"step has to be a positive integer constant");
1803 getOperandSegmentSizeAttr(),
1805 static_cast<int32_t>(ubOperands.size()),
1806 static_cast<int32_t>(iterArgs.size())}));
1808 for (
Value val : iterArgs)
1831 Value inductionVar =
1833 for (
Value val : iterArgs)
1834 bodyBlock.
addArgument(val.getType(), val.getLoc());
1839 if (iterArgs.empty() && !bodyBuilder) {
1840 ensureTerminator(*bodyRegion, builder, result.
location);
1841 }
else if (bodyBuilder) {
1844 bodyBuilder(builder, result.
location, inductionVar,
1850 int64_t ub, int64_t step,
ValueRange iterArgs,
1851 BodyBuilderFn bodyBuilder) {
1854 return build(builder, result, {}, lbMap, {}, ubMap, step, iterArgs,
1861 auto *body = getBody();
1862 if (body->getNumArguments() == 0 || !body->getArgument(0).getType().isIndex())
1863 return emitOpError(
"expected body to have a single index argument for the "
1864 "induction variable");
1868 if (getLowerBoundMap().getNumInputs() > 0)
1870 getLowerBoundMap().getNumDims())))
1873 if (getUpperBoundMap().getNumInputs() > 0)
1875 getUpperBoundMap().getNumDims())))
1878 unsigned opNumResults = getNumResults();
1879 if (opNumResults == 0)
1885 if (getNumIterOperands() != opNumResults)
1887 "mismatch between the number of loop-carried values and results");
1888 if (getNumRegionIterArgs() != opNumResults)
1890 "mismatch between the number of basic block args and results");
1900 bool failedToParsedMinMax =
1904 auto boundAttrStrName =
1905 isLower ? AffineForOp::getLowerBoundMapAttrName(result.
name)
1906 : AffineForOp::getUpperBoundMapAttrName(result.
name);
1913 if (!boundOpInfos.empty()) {
1915 if (boundOpInfos.size() > 1)
1917 "expected only one loop bound operand");
1942 if (
auto affineMapAttr = llvm::dyn_cast<AffineMapAttr>(boundAttr)) {
1943 unsigned currentNumOperands = result.
operands.size();
1948 auto map = affineMapAttr.getValue();
1952 "dim operand count and affine map dim count must match");
1954 unsigned numDimAndSymbolOperands =
1955 result.
operands.size() - currentNumOperands;
1956 if (numDims + map.
getNumSymbols() != numDimAndSymbolOperands)
1959 "symbol operand count and affine map symbol count must match");
1965 return p.
emitError(attrLoc,
"lower loop bound affine map with "
1966 "multiple results requires 'max' prefix");
1968 return p.
emitError(attrLoc,
"upper loop bound affine map with multiple "
1969 "results requires 'min' prefix");
1975 if (
auto integerAttr = llvm::dyn_cast<IntegerAttr>(boundAttr)) {
1985 "expected valid affine map representation for loop bounds");
1997 int64_t numOperands = result.
operands.size();
2000 int64_t numLbOperands = result.
operands.size() - numOperands;
2003 numOperands = result.
operands.size();
2006 int64_t numUbOperands = result.
operands.size() - numOperands;
2011 getStepAttrName(result.
name),
2015 IntegerAttr stepAttr;
2017 getStepAttrName(result.
name).data(),
2021 if (stepAttr.getValue().isNegative())
2024 "expected step to be representable as a positive signed integer");
2032 regionArgs.push_back(inductionVariable);
2040 for (
auto argOperandType :
2041 llvm::zip(llvm::drop_begin(regionArgs), operands, result.
types)) {
2042 Type type = std::get<2>(argOperandType);
2043 std::get<0>(argOperandType).type = type;
2051 getOperandSegmentSizeAttr(),
2053 static_cast<int32_t>(numUbOperands),
2054 static_cast<int32_t>(operands.size())}));
2058 if (regionArgs.size() != result.
types.size() + 1)
2061 "mismatch between the number of loop-carried values and results");
2065 AffineForOp::ensureTerminator(*body, builder, result.
location);
2087 if (
auto constExpr = dyn_cast<AffineConstantExpr>(expr)) {
2088 p << constExpr.getValue();
2096 if (dyn_cast<AffineSymbolExpr>(expr)) {
2112 unsigned AffineForOp::getNumIterOperands() {
2113 AffineMap lbMap = getLowerBoundMapAttr().getValue();
2114 AffineMap ubMap = getUpperBoundMapAttr().getValue();
2120 return cast<AffineYieldOp>(getBody()->getTerminator()).getOperandsMutable();
2132 if (getStepAsInt() != 1)
2133 p <<
" step " << getStepAsInt();
2135 bool printBlockTerminators =
false;
2136 if (getNumIterOperands() > 0) {
2138 auto regionArgs = getRegionIterArgs();
2139 auto operands = getInits();
2141 llvm::interleaveComma(llvm::zip(regionArgs, operands), p, [&](
auto it) {
2142 p << std::get<0>(it) <<
" = " << std::get<1>(it);
2144 p <<
") -> (" << getResultTypes() <<
")";
2145 printBlockTerminators =
true;
2150 printBlockTerminators);
2152 (*this)->getAttrs(),
2153 {getLowerBoundMapAttrName(getOperation()->getName()),
2154 getUpperBoundMapAttrName(getOperation()->getName()),
2155 getStepAttrName(getOperation()->getName()),
2156 getOperandSegmentSizeAttr()});
2161 auto foldLowerOrUpperBound = [&forOp](
bool lower) {
2165 auto boundOperands =
2166 lower ? forOp.getLowerBoundOperands() : forOp.getUpperBoundOperands();
2167 for (
auto operand : boundOperands) {
2170 operandConstants.push_back(operandCst);
2174 lower ? forOp.getLowerBoundMap() : forOp.getUpperBoundMap();
2176 "bound maps should have at least one result");
2182 assert(!foldedResults.empty() &&
"bounds should have at least one result");
2183 auto maxOrMin = llvm::cast<IntegerAttr>(foldedResults[0]).getValue();
2184 for (
unsigned i = 1, e = foldedResults.size(); i < e; i++) {
2185 auto foldedResult = llvm::cast<IntegerAttr>(foldedResults[i]).getValue();
2186 maxOrMin = lower ? llvm::APIntOps::smax(maxOrMin, foldedResult)
2187 : llvm::APIntOps::smin(maxOrMin, foldedResult);
2189 lower ? forOp.setConstantLowerBound(maxOrMin.getSExtValue())
2190 : forOp.setConstantUpperBound(maxOrMin.getSExtValue());
2195 bool folded =
false;
2196 if (!forOp.hasConstantLowerBound())
2197 folded |=
succeeded(foldLowerOrUpperBound(
true));
2200 if (!forOp.hasConstantUpperBound())
2201 folded |=
succeeded(foldLowerOrUpperBound(
false));
2210 auto lbMap = forOp.getLowerBoundMap();
2211 auto ubMap = forOp.getUpperBoundMap();
2212 auto prevLbMap = lbMap;
2213 auto prevUbMap = ubMap;
2226 if (lbMap == prevLbMap && ubMap == prevUbMap)
2229 if (lbMap != prevLbMap)
2230 forOp.setLowerBound(lbOperands, lbMap);
2231 if (ubMap != prevUbMap)
2232 forOp.setUpperBound(ubOperands, ubMap);
2238 static std::optional<uint64_t> getTrivialConstantTripCount(AffineForOp forOp) {
2239 int64_t step = forOp.getStepAsInt();
2240 if (!forOp.hasConstantBounds() || step <= 0)
2241 return std::nullopt;
2242 int64_t lb = forOp.getConstantLowerBound();
2243 int64_t ub = forOp.getConstantUpperBound();
2244 return ub - lb <= 0 ? 0 : (ub - lb + step - 1) / step;
2255 if (!llvm::hasSingleElement(*forOp.getBody()))
2257 if (forOp.getNumResults() == 0)
2259 std::optional<uint64_t> tripCount = getTrivialConstantTripCount(forOp);
2260 if (tripCount && *tripCount == 0) {
2263 rewriter.
replaceOp(forOp, forOp.getInits());
2267 auto yieldOp = cast<AffineYieldOp>(forOp.getBody()->getTerminator());
2268 auto iterArgs = forOp.getRegionIterArgs();
2269 bool hasValDefinedOutsideLoop =
false;
2270 bool iterArgsNotInOrder =
false;
2271 for (
unsigned i = 0, e = yieldOp->getNumOperands(); i < e; ++i) {
2272 Value val = yieldOp.getOperand(i);
2273 auto *iterArgIt = llvm::find(iterArgs, val);
2274 if (iterArgIt == iterArgs.end()) {
2276 assert(forOp.isDefinedOutsideOfLoop(val) &&
2277 "must be defined outside of the loop");
2278 hasValDefinedOutsideLoop =
true;
2279 replacements.push_back(val);
2281 unsigned pos = std::distance(iterArgs.begin(), iterArgIt);
2283 iterArgsNotInOrder =
true;
2284 replacements.push_back(forOp.getInits()[pos]);
2289 if (!tripCount.has_value() &&
2290 (hasValDefinedOutsideLoop || iterArgsNotInOrder))
2294 if (tripCount.has_value() && tripCount.value() >= 2 && iterArgsNotInOrder)
2296 rewriter.
replaceOp(forOp, replacements);
2304 results.
add<AffineForEmptyLoopFolder>(context);
2308 assert((point.
isParent() || point == getRegion()) &&
"invalid region point");
2315 void AffineForOp::getSuccessorRegions(
2317 assert((point.
isParent() || point == getRegion()) &&
"expected loop region");
2322 std::optional<uint64_t> tripCount = getTrivialConstantTripCount(*
this);
2323 if (point.
isParent() && tripCount.has_value()) {
2324 if (tripCount.value() > 0) {
2325 regions.push_back(
RegionSuccessor(&getRegion(), getRegionIterArgs()));
2328 if (tripCount.value() == 0) {
2336 if (!point.
isParent() && tripCount && *tripCount == 1) {
2343 regions.push_back(
RegionSuccessor(&getRegion(), getRegionIterArgs()));
2349 std::optional<uint64_t> tripCount = getTrivialConstantTripCount(op);
2350 return tripCount && *tripCount == 0;
2363 results.assign(getInits().begin(), getInits().end());
2379 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
2380 getLowerBoundOperandsMutable().assign(lbOperands);
2381 setLowerBoundMap(map);
2386 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
2387 getUpperBoundOperandsMutable().assign(ubOperands);
2388 setUpperBoundMap(map);
2391 bool AffineForOp::hasConstantLowerBound() {
2392 return getLowerBoundMap().isSingleConstant();
2395 bool AffineForOp::hasConstantUpperBound() {
2396 return getUpperBoundMap().isSingleConstant();
2399 int64_t AffineForOp::getConstantLowerBound() {
2400 return getLowerBoundMap().getSingleConstantResult();
2403 int64_t AffineForOp::getConstantUpperBound() {
2404 return getUpperBoundMap().getSingleConstantResult();
2407 void AffineForOp::setConstantLowerBound(int64_t value) {
2411 void AffineForOp::setConstantUpperBound(int64_t value) {
2415 AffineForOp::operand_range AffineForOp::getControlOperands() {
2420 bool AffineForOp::matchingBoundOperandList() {
2421 auto lbMap = getLowerBoundMap();
2422 auto ubMap = getUpperBoundMap();
2428 for (
unsigned i = 0, e = lbMap.
getNumInputs(); i < e; i++) {
2430 if (getOperand(i) != getOperand(numOperands + i))
2438 std::optional<Value> AffineForOp::getSingleInductionVar() {
2439 return getInductionVar();
2442 std::optional<OpFoldResult> AffineForOp::getSingleLowerBound() {
2443 if (!hasConstantLowerBound())
2444 return std::nullopt;
2446 return OpFoldResult(b.getI64IntegerAttr(getConstantLowerBound()));
2449 std::optional<OpFoldResult> AffineForOp::getSingleStep() {
2451 return OpFoldResult(b.getI64IntegerAttr(getStepAsInt()));
2454 std::optional<OpFoldResult> AffineForOp::getSingleUpperBound() {
2455 if (!hasConstantUpperBound())
2456 return std::nullopt;
2458 return OpFoldResult(b.getI64IntegerAttr(getConstantUpperBound()));
2463 bool replaceInitOperandUsesInLoop,
2468 auto inits = llvm::to_vector(getInits());
2469 inits.append(newInitOperands.begin(), newInitOperands.end());
2470 AffineForOp newLoop = rewriter.
create<AffineForOp>(
2475 auto yieldOp = cast<AffineYieldOp>(getBody()->getTerminator());
2477 newLoop.getBody()->getArguments().take_back(newInitOperands.size());
2482 newYieldValuesFn(rewriter, getLoc(), newIterArgs);
2483 assert(newInitOperands.size() == newYieldedValues.size() &&
2484 "expected as many new yield values as new iter operands");
2486 yieldOp.getOperandsMutable().append(newYieldedValues);
2491 rewriter.
mergeBlocks(getBody(), newLoop.getBody(),
2492 newLoop.getBody()->getArguments().take_front(
2493 getBody()->getNumArguments()));
2495 if (replaceInitOperandUsesInLoop) {
2498 for (
auto it : llvm::zip(newInitOperands, newIterArgs)) {
2509 newLoop->getResults().take_front(getNumResults()));
2510 return cast<LoopLikeOpInterface>(newLoop.getOperation());
2538 auto ivArg = llvm::dyn_cast<BlockArgument>(val);
2539 if (!ivArg || !ivArg.getOwner())
2540 return AffineForOp();
2541 auto *containingInst = ivArg.getOwner()->getParent()->getParentOp();
2542 if (
auto forOp = dyn_cast<AffineForOp>(containingInst))
2544 return forOp.getInductionVar() == val ? forOp : AffineForOp();
2545 return AffineForOp();
2549 auto ivArg = llvm::dyn_cast<BlockArgument>(val);
2550 if (!ivArg || !ivArg.getOwner())
2553 auto parallelOp = dyn_cast<AffineParallelOp>(containingOp);
2554 if (parallelOp && llvm::is_contained(parallelOp.getIVs(), val))
2563 ivs->reserve(forInsts.size());
2564 for (
auto forInst : forInsts)
2565 ivs->push_back(forInst.getInductionVar());
2570 ivs.reserve(affineOps.size());
2573 if (
auto forOp = dyn_cast<AffineForOp>(op))
2574 ivs.push_back(forOp.getInductionVar());
2575 else if (
auto parallelOp = dyn_cast<AffineParallelOp>(op))
2576 for (
size_t i = 0; i < parallelOp.getBody()->getNumArguments(); i++)
2577 ivs.push_back(parallelOp.getBody()->getArgument(i));
2583 template <
typename BoundListTy,
typename LoopCreatorTy>
2588 LoopCreatorTy &&loopCreatorFn) {
2589 assert(lbs.size() == ubs.size() &&
"Mismatch in number of arguments");
2590 assert(lbs.size() == steps.size() &&
"Mismatch in number of arguments");
2602 ivs.reserve(lbs.size());
2603 for (
unsigned i = 0, e = lbs.size(); i < e; ++i) {
2609 if (i == e - 1 && bodyBuilderFn) {
2611 bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
2613 nestedBuilder.
create<AffineYieldOp>(nestedLoc);
2618 auto loop = loopCreatorFn(builder, loc, lbs[i], ubs[i], steps[i], loopBody);
2626 int64_t ub, int64_t step,
2627 AffineForOp::BodyBuilderFn bodyBuilderFn) {
2628 return builder.
create<AffineForOp>(loc, lb, ub, step,
2629 std::nullopt, bodyBuilderFn);
2636 AffineForOp::BodyBuilderFn bodyBuilderFn) {
2639 if (lbConst && ubConst)
2641 ubConst.value(), step, bodyBuilderFn);
2644 std::nullopt, bodyBuilderFn);
2674 if (ifOp.getElseRegion().empty() ||
2675 !llvm::hasSingleElement(*ifOp.getElseBlock()) || ifOp.getNumResults())
2693 auto isTriviallyFalse = [](
IntegerSet iSet) {
2694 return iSet.isEmptyIntegerSet();
2698 return (iSet.getNumEqualities() == 1 && iSet.getNumInequalities() == 0 &&
2699 iSet.getConstraint(0) == 0);
2702 IntegerSet affineIfConditions = op.getIntegerSet();
2704 if (isTriviallyFalse(affineIfConditions)) {
2714 blockToMove = op.getElseBlock();
2715 }
else if (isTriviallyTrue(affineIfConditions)) {
2716 blockToMove = op.getThenBlock();
2734 rewriter.
eraseOp(blockToMoveTerminator);
2742 void AffineIfOp::getSuccessorRegions(
2751 if (getElseRegion().empty()) {
2752 regions.push_back(getResults());
2768 auto conditionAttr =
2769 (*this)->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName());
2771 return emitOpError(
"requires an integer set attribute named 'condition'");
2774 IntegerSet condition = conditionAttr.getValue();
2776 return emitOpError(
"operand count and condition integer set dimension and "
2777 "symbol count must match");
2789 IntegerSetAttr conditionAttr;
2792 AffineIfOp::getConditionAttrStrName(),
2798 auto set = conditionAttr.getValue();
2799 if (set.getNumDims() != numDims)
2802 "dim operand count and integer set dim count must match");
2803 if (numDims + set.getNumSymbols() != result.
operands.size())
2806 "symbol operand count and integer set symbol count must match");
2820 AffineIfOp::ensureTerminator(*thenRegion, parser.
getBuilder(),
2827 AffineIfOp::ensureTerminator(*elseRegion, parser.
getBuilder(),
2839 auto conditionAttr =
2840 (*this)->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName());
2841 p <<
" " << conditionAttr;
2843 conditionAttr.getValue().getNumDims(), p);
2850 auto &elseRegion = this->getElseRegion();
2851 if (!elseRegion.
empty()) {
2860 getConditionAttrStrName());
2865 ->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName())
2869 void AffineIfOp::setIntegerSet(
IntegerSet newSet) {
2875 (*this)->setOperands(operands);
2880 bool withElseRegion) {
2881 assert(resultTypes.empty() || withElseRegion);
2888 if (resultTypes.empty())
2889 AffineIfOp::ensureTerminator(*thenRegion, builder, result.
location);
2892 if (withElseRegion) {
2894 if (resultTypes.empty())
2895 AffineIfOp::ensureTerminator(*elseRegion, builder, result.
location);
2901 AffineIfOp::build(builder, result, {}, set, args,
2916 if (llvm::none_of(operands,
2927 auto set = getIntegerSet();
2933 if (getIntegerSet() == set && llvm::equal(operands, getOperands()))
2936 setConditional(set, operands);
2942 results.
add<SimplifyDeadElse, AlwaysTrueOrFalseIf>(context);
2951 assert(operands.size() == 1 + map.
getNumInputs() &&
"inconsistent operands");
2955 auto memrefType = llvm::cast<MemRefType>(operands[0].getType());
2956 result.
types.push_back(memrefType.getElementType());
2961 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
2964 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
2966 result.
types.push_back(memrefType.getElementType());
2971 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
2972 int64_t rank = memrefType.getRank();
2977 build(builder, result, memref, map, indices);
2986 AffineMapAttr mapAttr;
2991 AffineLoadOp::getMapAttrStrName(),
3001 p <<
" " << getMemRef() <<
'[';
3002 if (AffineMapAttr mapAttr =
3003 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
3007 {getMapAttrStrName()});
3016 MemRefType memrefType,
unsigned numIndexOperands) {
3019 return op->
emitOpError(
"affine map num results must equal memref rank");
3021 return op->
emitOpError(
"expects as many subscripts as affine map inputs");
3024 for (
auto idx : mapOperands) {
3025 if (!idx.getType().isIndex())
3026 return op->
emitOpError(
"index to load must have 'index' type");
3029 "index must be a valid dimension or symbol identifier");
3037 if (getType() != memrefType.getElementType())
3038 return emitOpError(
"result type must match element type of memref");
3042 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
3043 getMapOperands(), memrefType,
3044 getNumOperands() - 1)))
3052 results.
add<SimplifyAffineOp<AffineLoadOp>>(context);
3061 auto getGlobalOp = getMemref().getDefiningOp<memref::GetGlobalOp>();
3068 auto global = dyn_cast_or_null<memref::GlobalOp>(
3075 llvm::dyn_cast_or_null<DenseElementsAttr>(global.getConstantInitValue());
3079 if (
auto splatAttr = llvm::dyn_cast<SplatElementsAttr>(cstAttr))
3080 return splatAttr.getSplatValue<
Attribute>();
3082 if (!getAffineMap().isConstant())
3084 auto indices = llvm::to_vector<4>(
3085 llvm::map_range(getAffineMap().getConstantResults(),
3086 [](int64_t v) -> uint64_t {
return v; }));
3087 return cstAttr.getValues<
Attribute>()[indices];
3097 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
3108 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
3109 int64_t rank = memrefType.getRank();
3114 build(builder, result, valueToStore, memref, map, indices);
3123 AffineMapAttr mapAttr;
3128 mapOperands, mapAttr, AffineStoreOp::getMapAttrStrName(),
3139 p <<
" " << getValueToStore();
3140 p <<
", " << getMemRef() <<
'[';
3141 if (AffineMapAttr mapAttr =
3142 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
3146 {getMapAttrStrName()});
3153 if (getValueToStore().getType() != memrefType.getElementType())
3155 "value to store must have the same type as memref element type");
3159 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
3160 getMapOperands(), memrefType,
3161 getNumOperands() - 2)))
3169 results.
add<SimplifyAffineOp<AffineStoreOp>>(context);
3182 template <
typename T>
3186 op.getMap().getNumDims() + op.getMap().getNumSymbols())
3188 "operand count and affine map dimension and symbol count must match");
3192 template <
typename T>
3194 p <<
' ' << op->
getAttr(T::getMapAttrStrName());
3196 unsigned numDims = op.getMap().getNumDims();
3197 p <<
'(' << operands.take_front(numDims) <<
')';
3199 if (operands.size() != numDims)
3200 p <<
'[' << operands.drop_front(numDims) <<
']';
3202 {T::getMapAttrStrName()});
3205 template <
typename T>
3212 AffineMapAttr mapAttr;
3228 template <
typename T>
3230 static_assert(llvm::is_one_of<T, AffineMinOp, AffineMaxOp>::value,
3231 "expected affine min or max op");
3237 auto foldedMap = op.getMap().partialConstantFold(operands, &results);
3239 if (foldedMap.getNumSymbols() == 1 && foldedMap.isSymbolIdentity())
3243 if (results.empty()) {
3245 if (foldedMap == op.getMap())
3252 auto resultIt = std::is_same<T, AffineMinOp>::value
3253 ? std::min_element(results.begin(), results.end())
3254 : std::max_element(results.begin(), results.end());
3255 if (resultIt == results.end())
3261 template <
typename T>
3267 AffineMap oldMap = affineOp.getAffineMap();
3273 if (!llvm::is_contained(newExprs, expr))
3274 newExprs.push_back(expr);
3304 template <
typename T>
3310 AffineMap oldMap = affineOp.getAffineMap();
3312 affineOp.getMapOperands().take_front(oldMap.
getNumDims());
3314 affineOp.getMapOperands().take_back(oldMap.
getNumSymbols());
3316 auto newDimOperands = llvm::to_vector<8>(dimOperands);
3317 auto newSymOperands = llvm::to_vector<8>(symOperands);
3325 if (
auto symExpr = dyn_cast<AffineSymbolExpr>(expr)) {
3326 Value symValue = symOperands[symExpr.getPosition()];
3328 producerOps.push_back(producerOp);
3331 }
else if (
auto dimExpr = dyn_cast<AffineDimExpr>(expr)) {
3332 Value dimValue = dimOperands[dimExpr.getPosition()];
3334 producerOps.push_back(producerOp);
3341 newExprs.push_back(expr);
3344 if (producerOps.empty())
3351 for (T producerOp : producerOps) {
3352 AffineMap producerMap = producerOp.getAffineMap();
3353 unsigned numProducerDims = producerMap.
getNumDims();
3358 producerOp.getMapOperands().take_front(numProducerDims);
3360 producerOp.getMapOperands().take_back(numProducerSyms);
3361 newDimOperands.append(dimValues.begin(), dimValues.end());
3362 newSymOperands.append(symValues.begin(), symValues.end());
3366 newExprs.push_back(expr.shiftDims(numProducerDims, numUsedDims)
3367 .shiftSymbols(numProducerSyms, numUsedSyms));
3370 numUsedDims += numProducerDims;
3371 numUsedSyms += numProducerSyms;
3377 llvm::to_vector<8>(llvm::concat<Value>(newDimOperands, newSymOperands));
3396 if (!resultExpr.isPureAffine())
3401 if (
failed(flattenResult))
3414 if (llvm::is_sorted(flattenedExprs))
3419 llvm::to_vector(llvm::seq<unsigned>(0, map.
getNumResults()));
3420 llvm::sort(resultPermutation, [&](
unsigned lhs,
unsigned rhs) {
3421 return flattenedExprs[lhs] < flattenedExprs[rhs];
3424 for (
unsigned idx : resultPermutation)
3445 template <
typename T>
3451 AffineMap map = affineOp.getAffineMap();
3459 template <
typename T>
3465 if (affineOp.getMap().getNumResults() != 1)
3468 affineOp.getOperands());
3496 return parseAffineMinMaxOp<AffineMinOp>(parser, result);
3524 return parseAffineMinMaxOp<AffineMaxOp>(parser, result);
3543 IntegerAttr hintInfo;
3545 StringRef readOrWrite, cacheType;
3547 AffineMapAttr mapAttr;
3551 AffinePrefetchOp::getMapAttrStrName(),
3557 AffinePrefetchOp::getLocalityHintAttrStrName(),
3567 if (!readOrWrite.equals(
"read") && !readOrWrite.equals(
"write"))
3569 "rw specifier has to be 'read' or 'write'");
3571 AffinePrefetchOp::getIsWriteAttrStrName(),
3574 if (!cacheType.equals(
"data") && !cacheType.equals(
"instr"))
3576 "cache type has to be 'data' or 'instr'");
3579 AffinePrefetchOp::getIsDataCacheAttrStrName(),
3586 p <<
" " << getMemref() <<
'[';
3587 AffineMapAttr mapAttr =
3588 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName());
3591 p <<
']' <<
", " << (getIsWrite() ?
"write" :
"read") <<
", "
3592 <<
"locality<" << getLocalityHint() <<
">, "
3593 << (getIsDataCache() ?
"data" :
"instr");
3595 (*this)->getAttrs(),
3596 {getMapAttrStrName(), getLocalityHintAttrStrName(),
3597 getIsDataCacheAttrStrName(), getIsWriteAttrStrName()});
3602 auto mapAttr = (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName());
3606 return emitOpError(
"affine.prefetch affine map num results must equal"
3609 return emitOpError(
"too few operands");
3611 if (getNumOperands() != 1)
3612 return emitOpError(
"too few operands");
3616 for (
auto idx : getMapOperands()) {
3619 "index must be a valid dimension or symbol identifier");
3627 results.
add<SimplifyAffineOp<AffinePrefetchOp>>(context);
3645 auto ubs = llvm::to_vector<4>(llvm::map_range(ranges, [&](int64_t value) {
3649 build(builder, result, resultTypes, reductions, lbs, {}, ubs,
3659 assert(llvm::all_of(lbMaps,
3661 return m.
getNumDims() == lbMaps[0].getNumDims() &&
3664 "expected all lower bounds maps to have the same number of dimensions "
3666 assert(llvm::all_of(ubMaps,
3668 return m.
getNumDims() == ubMaps[0].getNumDims() &&
3671 "expected all upper bounds maps to have the same number of dimensions "
3673 assert((lbMaps.empty() || lbMaps[0].getNumInputs() == lbArgs.size()) &&
3674 "expected lower bound maps to have as many inputs as lower bound "
3676 assert((ubMaps.empty() || ubMaps[0].getNumInputs() == ubArgs.size()) &&
3677 "expected upper bound maps to have as many inputs as upper bound "
3684 for (arith::AtomicRMWKind reduction : reductions)
3685 reductionAttrs.push_back(
3697 groups.reserve(groups.size() + maps.size());
3698 exprs.reserve(maps.size());
3703 return AffineMap::get(maps[0].getNumDims(), maps[0].getNumSymbols(), exprs,
3709 AffineMap lbMap = concatMapsSameInput(lbMaps, lbGroups);
3710 AffineMap ubMap = concatMapsSameInput(ubMaps, ubGroups);
3725 auto *body =
new Block();
3727 for (
unsigned i = 0, e = steps.size(); i < e; ++i)
3730 if (resultTypes.empty())
3731 ensureTerminator(*bodyRegion, builder, result.
location);
3735 return {&getRegion()};
3738 unsigned AffineParallelOp::getNumDims() {
return getSteps().size(); }
3740 AffineParallelOp::operand_range AffineParallelOp::getLowerBoundsOperands() {
3741 return getOperands().take_front(getLowerBoundsMap().getNumInputs());
3744 AffineParallelOp::operand_range AffineParallelOp::getUpperBoundsOperands() {
3745 return getOperands().drop_front(getLowerBoundsMap().getNumInputs());
3748 AffineMap AffineParallelOp::getLowerBoundMap(
unsigned pos) {
3749 auto values = getLowerBoundsGroups().getValues<int32_t>();
3751 for (
unsigned i = 0; i < pos; ++i)
3753 return getLowerBoundsMap().getSliceMap(start, values[pos]);
3756 AffineMap AffineParallelOp::getUpperBoundMap(
unsigned pos) {
3757 auto values = getUpperBoundsGroups().getValues<int32_t>();
3759 for (
unsigned i = 0; i < pos; ++i)
3761 return getUpperBoundsMap().getSliceMap(start, values[pos]);
3765 return AffineValueMap(getLowerBoundsMap(), getLowerBoundsOperands());
3769 return AffineValueMap(getUpperBoundsMap(), getUpperBoundsOperands());
3772 std::optional<SmallVector<int64_t, 8>> AffineParallelOp::getConstantRanges() {
3773 if (hasMinMaxBounds())
3774 return std::nullopt;
3779 AffineValueMap::difference(getUpperBoundsValueMap(), getLowerBoundsValueMap(),
3782 for (
unsigned i = 0, e = rangesValueMap.
getNumResults(); i < e; ++i) {
3783 auto expr = rangesValueMap.
getResult(i);
3784 auto cst = dyn_cast<AffineConstantExpr>(expr);
3786 return std::nullopt;
3787 out.push_back(cst.getValue());
3792 Block *AffineParallelOp::getBody() {
return &getRegion().
front(); }
3794 OpBuilder AffineParallelOp::getBodyBuilder() {
3795 return OpBuilder(getBody(), std::prev(getBody()->end()));
3800 "operands to map must match number of inputs");
3802 auto ubOperands = getUpperBoundsOperands();
3805 newOperands.append(ubOperands.begin(), ubOperands.end());
3806 (*this)->setOperands(newOperands);
3813 "operands to map must match number of inputs");
3816 newOperands.append(ubOperands.begin(), ubOperands.end());
3817 (*this)->setOperands(newOperands);
3823 setStepsAttr(getBodyBuilder().getI64ArrayAttr(newSteps));
3828 arith::AtomicRMWKind op) {
3830 case arith::AtomicRMWKind::addf:
3831 return isa<FloatType>(resultType);
3832 case arith::AtomicRMWKind::addi:
3833 return isa<IntegerType>(resultType);
3834 case arith::AtomicRMWKind::assign:
3836 case arith::AtomicRMWKind::mulf:
3837 return isa<FloatType>(resultType);
3838 case arith::AtomicRMWKind::muli:
3839 return isa<IntegerType>(resultType);
3840 case arith::AtomicRMWKind::maximumf:
3841 return isa<FloatType>(resultType);
3842 case arith::AtomicRMWKind::minimumf:
3843 return isa<FloatType>(resultType);
3844 case arith::AtomicRMWKind::maxs: {
3845 auto intType = llvm::dyn_cast<IntegerType>(resultType);
3846 return intType && intType.isSigned();
3848 case arith::AtomicRMWKind::mins: {
3849 auto intType = llvm::dyn_cast<IntegerType>(resultType);
3850 return intType && intType.isSigned();
3852 case arith::AtomicRMWKind::maxu: {
3853 auto intType = llvm::dyn_cast<IntegerType>(resultType);
3854 return intType && intType.isUnsigned();
3856 case arith::AtomicRMWKind::minu: {
3857 auto intType = llvm::dyn_cast<IntegerType>(resultType);
3858 return intType && intType.isUnsigned();
3860 case arith::AtomicRMWKind::ori:
3861 return isa<IntegerType>(resultType);
3862 case arith::AtomicRMWKind::andi:
3863 return isa<IntegerType>(resultType);
3870 auto numDims = getNumDims();
3873 getSteps().size() != numDims || getBody()->getNumArguments() != numDims) {
3874 return emitOpError() <<
"the number of region arguments ("
3875 << getBody()->getNumArguments()
3876 <<
") and the number of map groups for lower ("
3877 << getLowerBoundsGroups().getNumElements()
3878 <<
") and upper bound ("
3879 << getUpperBoundsGroups().getNumElements()
3880 <<
"), and the number of steps (" << getSteps().size()
3881 <<
") must all match";
3884 unsigned expectedNumLBResults = 0;
3885 for (APInt v : getLowerBoundsGroups())
3886 expectedNumLBResults += v.getZExtValue();
3887 if (expectedNumLBResults != getLowerBoundsMap().getNumResults())
3888 return emitOpError() <<
"expected lower bounds map to have "
3889 << expectedNumLBResults <<
" results";
3890 unsigned expectedNumUBResults = 0;
3891 for (APInt v : getUpperBoundsGroups())
3892 expectedNumUBResults += v.getZExtValue();
3893 if (expectedNumUBResults != getUpperBoundsMap().getNumResults())
3894 return emitOpError() <<
"expected upper bounds map to have "
3895 << expectedNumUBResults <<
" results";
3897 if (getReductions().size() != getNumResults())
3898 return emitOpError(
"a reduction must be specified for each output");
3904 auto intAttr = llvm::dyn_cast<IntegerAttr>(attr);
3905 if (!intAttr || !arith::symbolizeAtomicRMWKind(intAttr.getInt()))
3906 return emitOpError(
"invalid reduction attribute");
3907 auto kind = arith::symbolizeAtomicRMWKind(intAttr.getInt()).value();
3909 return emitOpError(
"result type cannot match reduction attribute");
3915 getLowerBoundsMap().getNumDims())))
3919 getUpperBoundsMap().getNumDims())))
3926 auto newMap = getAffineMap();
3928 if (newMap == getAffineMap() && newOperands == operands)
3930 reset(newMap, newOperands);
3943 if (!lbCanonicalized && !ubCanonicalized)
3946 if (lbCanonicalized)
3948 if (ubCanonicalized)
3966 StringRef keyword) {
3969 ValueRange dimOperands = operands.take_front(numDims);
3970 ValueRange symOperands = operands.drop_front(numDims);
3972 for (llvm::APInt groupSize : group) {
3976 unsigned size = groupSize.getZExtValue();
3981 p << keyword <<
'(';
3991 p <<
" (" << getBody()->getArguments() <<
") = (";
3993 getLowerBoundsOperands(),
"max");
3996 getUpperBoundsOperands(),
"min");
3999 bool elideSteps = llvm::all_of(steps, [](int64_t step) {
return step == 1; });
4002 llvm::interleaveComma(steps, p);
4005 if (getNumResults()) {
4007 llvm::interleaveComma(getReductions(), p, [&](
auto &attr) {
4008 arith::AtomicRMWKind sym = *arith::symbolizeAtomicRMWKind(
4009 llvm::cast<IntegerAttr>(attr).getInt());
4010 p <<
"\"" << arith::stringifyAtomicRMWKind(sym) <<
"\"";
4012 p <<
") -> (" << getResultTypes() <<
")";
4019 (*this)->getAttrs(),
4020 {AffineParallelOp::getReductionsAttrStrName(),
4021 AffineParallelOp::getLowerBoundsMapAttrStrName(),
4022 AffineParallelOp::getLowerBoundsGroupsAttrStrName(),
4023 AffineParallelOp::getUpperBoundsMapAttrStrName(),
4024 AffineParallelOp::getUpperBoundsGroupsAttrStrName(),
4025 AffineParallelOp::getStepsAttrStrName()});
4038 "expected operands to be dim or symbol expression");
4041 for (
const auto &list : operands) {
4045 for (
Value operand : valueOperands) {
4046 unsigned pos = std::distance(uniqueOperands.begin(),
4047 llvm::find(uniqueOperands, operand));
4048 if (pos == uniqueOperands.size())
4049 uniqueOperands.push_back(operand);
4050 replacements.push_back(
4060 enum class MinMaxKind { Min, Max };
4084 const llvm::StringLiteral tmpAttrStrName =
"__pseudo_bound_map";
4086 StringRef mapName = kind == MinMaxKind::Min
4087 ? AffineParallelOp::getUpperBoundsMapAttrStrName()
4088 : AffineParallelOp::getLowerBoundsMapAttrStrName();
4089 StringRef groupsName =
4090 kind == MinMaxKind::Min
4091 ? AffineParallelOp::getUpperBoundsGroupsAttrStrName()
4092 : AffineParallelOp::getLowerBoundsGroupsAttrStrName();
4109 auto parseOperands = [&]() {
4111 kind == MinMaxKind::Min ?
"min" :
"max"))) {
4112 mapOperands.clear();
4119 llvm::append_range(flatExprs, map.getValue().getResults());
4121 auto dimsRef = operandsRef.take_front(map.getValue().getNumDims());
4124 auto symsRef = operandsRef.drop_front(map.getValue().getNumDims());
4127 flatDimOperands.append(map.getValue().getNumResults(), dims);
4128 flatSymOperands.append(map.getValue().getNumResults(), syms);
4129 numMapsPerGroup.push_back(map.getValue().getNumResults());
4132 flatSymOperands.emplace_back(),
4133 flatExprs.emplace_back())))
4135 numMapsPerGroup.push_back(1);
4142 unsigned totalNumDims = 0;
4143 unsigned totalNumSyms = 0;
4144 for (
unsigned i = 0, e = flatExprs.size(); i < e; ++i) {
4145 unsigned numDims = flatDimOperands[i].size();
4146 unsigned numSyms = flatSymOperands[i].size();
4147 flatExprs[i] = flatExprs[i]
4148 .shiftDims(numDims, totalNumDims)
4149 .shiftSymbols(numSyms, totalNumSyms);
4150 totalNumDims += numDims;
4151 totalNumSyms += numSyms;
4163 result.
operands.append(dimOperands.begin(), dimOperands.end());
4164 result.
operands.append(symOperands.begin(), symOperands.end());
4167 auto flatMap =
AffineMap::get(totalNumDims, totalNumSyms, flatExprs,
4169 flatMap = flatMap.replaceDimsAndSymbols(
4170 dimRplacements, symRepacements, dimOperands.size(), symOperands.size());
4194 AffineMapAttr stepsMapAttr;
4199 result.
addAttribute(AffineParallelOp::getStepsAttrStrName(),
4203 AffineParallelOp::getStepsAttrStrName(),
4210 auto stepsMap = stepsMapAttr.getValue();
4211 for (
const auto &result : stepsMap.getResults()) {
4212 auto constExpr = dyn_cast<AffineConstantExpr>(result);
4215 "steps must be constant integers");
4216 steps.push_back(constExpr.getValue());
4218 result.
addAttribute(AffineParallelOp::getStepsAttrStrName(),
4238 std::optional<arith::AtomicRMWKind> reduction =
4239 arith::symbolizeAtomicRMWKind(attrVal.getValue());
4241 return parser.
emitError(loc,
"invalid reduction value: ") << attrVal;
4242 reductions.push_back(
4250 result.
addAttribute(AffineParallelOp::getReductionsAttrStrName(),
4259 for (
auto &iv : ivs)
4260 iv.type = indexType;
4266 AffineParallelOp::ensureTerminator(*body, builder, result.
location);
4275 auto *parentOp = (*this)->getParentOp();
4276 auto results = parentOp->getResults();
4277 auto operands = getOperands();
4279 if (!isa<AffineParallelOp, AffineIfOp, AffineForOp>(parentOp))
4280 return emitOpError() <<
"only terminates affine.if/for/parallel regions";
4281 if (parentOp->getNumResults() != getNumOperands())
4282 return emitOpError() <<
"parent of yield must have same number of "
4283 "results as the yield operands";
4284 for (
auto it : llvm::zip(results, operands)) {
4285 if (std::get<0>(it).getType() != std::get<1>(it).getType())
4286 return emitOpError() <<
"types mismatch between yield op and its parent";
4299 assert(operands.size() == 1 + map.
getNumInputs() &&
"inconsistent operands");
4303 result.
types.push_back(resultType);
4307 VectorType resultType,
Value memref,
4309 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
4313 result.
types.push_back(resultType);
4317 VectorType resultType,
Value memref,
4319 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
4320 int64_t rank = memrefType.getRank();
4325 build(builder, result, resultType, memref, map, indices);
4328 void AffineVectorLoadOp::getCanonicalizationPatterns(
RewritePatternSet &results,
4330 results.
add<SimplifyAffineOp<AffineVectorLoadOp>>(context);
4338 MemRefType memrefType;
4339 VectorType resultType;
4341 AffineMapAttr mapAttr;
4346 AffineVectorLoadOp::getMapAttrStrName(),
4357 p <<
" " << getMemRef() <<
'[';
4358 if (AffineMapAttr mapAttr =
4359 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
4363 {getMapAttrStrName()});
4369 VectorType vectorType) {
4371 if (memrefType.getElementType() != vectorType.getElementType())
4373 "requires memref and vector types of the same elemental type");
4381 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
4382 getMapOperands(), memrefType,
4383 getNumOperands() - 1)))
4399 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
4410 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
4411 int64_t rank = memrefType.getRank();
4416 build(builder, result, valueToStore, memref, map, indices);
4418 void AffineVectorStoreOp::getCanonicalizationPatterns(
4420 results.
add<SimplifyAffineOp<AffineVectorStoreOp>>(context);
4427 MemRefType memrefType;
4428 VectorType resultType;
4431 AffineMapAttr mapAttr;
4437 AffineVectorStoreOp::getMapAttrStrName(),
4448 p <<
" " << getValueToStore();
4449 p <<
", " << getMemRef() <<
'[';
4450 if (AffineMapAttr mapAttr =
4451 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
4455 {getMapAttrStrName()});
4456 p <<
" : " <<
getMemRefType() <<
", " << getValueToStore().getType();
4462 *
this, (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
4463 getMapOperands(), memrefType,
4464 getNumOperands() - 2)))
4485 if (staticDim.has_value())
4488 return llvm::dyn_cast_if_present<Value>(ofr);
4494 if (getBasis().empty())
4495 return emitOpError(
"basis should not be empty");
4496 if (getNumResults() != getBasis().size())
4497 return emitOpError(
"should return an index for each basis element");
4505 #define GET_OP_CLASSES
4506 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc"
static AffineForOp buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb, int64_t ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds known to be constants.
static bool hasTrivialZeroTripCount(AffineForOp op)
Returns true if the affine.for has zero iterations in trivial cases.
static void composeMultiResultAffineMap(AffineMap &map, SmallVectorImpl< Value > &operands)
Composes the given affine map with the given list of operands, pulling in the maps from any affine....
static void printAffineMinMaxOp(OpAsmPrinter &p, T op)
static bool isResultTypeMatchAtomicRMWKind(Type resultType, arith::AtomicRMWKind op)
static bool remainsLegalAfterInline(Value value, Region *src, Region *dest, const IRMapping &mapping, function_ref< bool(Value, Region *)> legalityCheck)
Checks if value known to be a legal affine dimension or symbol in src region remains legal if the ope...
static void printMinMaxBound(OpAsmPrinter &p, AffineMapAttr mapAttr, DenseIntElementsAttr group, ValueRange operands, StringRef keyword)
Prints a lower(upper) bound of an affine parallel loop with max(min) conditions in it.
static void LLVM_ATTRIBUTE_UNUSED simplifyMapWithOperands(AffineMap &map, ArrayRef< Value > operands)
Simplify the map while exploiting information on the values in operands.
static OpFoldResult foldMinMaxOp(T op, ArrayRef< Attribute > operands)
Fold an affine min or max operation with the given operands.
static LogicalResult canonicalizeLoopBounds(AffineForOp forOp)
Canonicalize the bounds of the given loop.
static void simplifyExprAndOperands(AffineExpr &expr, unsigned numDims, unsigned numSymbols, ArrayRef< Value > operands)
Simplify expr while exploiting information from the values in operands.
static bool isValidAffineIndexOperand(Value value, Region *region)
static void canonicalizeMapOrSetAndOperands(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
static void composeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Iterate over operands and fold away all those produced by an AffineApplyOp iteratively.
static std::optional< int64_t > getUpperBound(Value iv)
Gets the constant upper bound on an affine.for iv.
static ParseResult parseBound(bool isLower, OperationState &result, OpAsmParser &p)
Parse a for operation loop bounds.
static std::optional< int64_t > getLowerBound(Value iv)
Gets the constant lower bound on an iv.
static void composeSetAndOperands(IntegerSet &set, SmallVectorImpl< Value > &operands)
Compose any affine.apply ops feeding into operands of the integer set set by composing the maps of su...
static LogicalResult replaceDimOrSym(AffineMap *map, unsigned dimOrSymbolPosition, SmallVectorImpl< Value > &dims, SmallVectorImpl< Value > &syms)
Replace all occurrences of AffineExpr at position pos in map by the defining AffineApplyOp expression...
static void canonicalizePromotedSymbols(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
static LogicalResult verifyVectorMemoryOp(Operation *op, MemRefType memrefType, VectorType vectorType)
Verify common invariants of affine.vector_load and affine.vector_store.
static void simplifyMinOrMaxExprWithOperands(AffineMap &map, ArrayRef< Value > operands, bool isMax)
Simplify the expressions in map while making use of lower or upper bounds of its operands.
static ParseResult parseAffineMinMaxOp(OpAsmParser &parser, OperationState &result)
static bool isMemRefSizeValidSymbol(AnyMemRefDefOp memrefDefOp, unsigned index, Region *region)
Returns true if the 'index' dimension of the memref defined by memrefDefOp is a statically shaped one...
static bool isNonNegativeBoundedBy(AffineExpr e, ArrayRef< Value > operands, int64_t k)
Check if e is known to be: 0 <= e < k.
static ParseResult parseAffineMapWithMinMax(OpAsmParser &parser, OperationState &result, MinMaxKind kind)
Parses an affine map that can contain a min/max for groups of its results, e.g., max(expr-1,...
static AffineForOp buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds that may or may not be constants.
static void printDimAndSymbolList(Operation::operand_iterator begin, Operation::operand_iterator end, unsigned numDims, OpAsmPrinter &printer)
Prints dimension and symbol list.
static int64_t getLargestKnownDivisor(AffineExpr e, ArrayRef< Value > operands)
Returns the largest known divisor of e.
static OpTy makeComposedMinMax(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
static void buildAffineLoopNestImpl(OpBuilder &builder, Location loc, BoundListTy lbs, BoundListTy ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn, LoopCreatorTy &&loopCreatorFn)
Builds an affine loop nest, using "loopCreatorFn" to create individual loop operations.
static LogicalResult foldLoopBounds(AffineForOp forOp)
Fold the constant bounds of a loop.
static LogicalResult verifyDimAndSymbolIdentifiers(OpTy &op, Operation::operand_range operands, unsigned numDims)
Utility function to verify that a set of operands are valid dimension and symbol identifiers.
static OpFoldResult makeComposedFoldedMinMax(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
static bool isDimOpValidSymbol(ShapedDimOpInterface dimOp, Region *region)
Returns true if the result of the dim op is a valid symbol for region.
static bool isQTimesDPlusR(AffineExpr e, ArrayRef< Value > operands, int64_t &div, AffineExpr "ientTimesDiv, AffineExpr &rem)
Check if expression e is of the form d*e_1 + e_2 where 0 <= e_2 < d.
static ParseResult deduplicateAndResolveOperands(OpAsmParser &parser, ArrayRef< SmallVector< OpAsmParser::UnresolvedOperand >> operands, SmallVectorImpl< Value > &uniqueOperands, SmallVectorImpl< AffineExpr > &replacements, AffineExprKind kind)
Given a list of lists of parsed operands, populates uniqueOperands with unique operands.
static LogicalResult verifyAffineMinMaxOp(T op)
static void printBound(AffineMapAttr boundMap, Operation::operand_range boundOperands, const char *prefix, OpAsmPrinter &p)
static LogicalResult verifyMemoryOpIndexing(Operation *op, AffineMapAttr mapAttr, Operation::operand_range mapOperands, MemRefType memrefType, unsigned numIndexOperands)
Verify common indexing invariants of affine.load, affine.store, affine.vector_load and affine....
static LogicalResult canonicalizeMapExprAndTermOrder(AffineMap &map)
Canonicalize the result expression order of an affine map and return success if the order changed.
static Operation * materializeConstant(Dialect *dialect, OpBuilder &builder, Attribute value, Type type, Location loc)
A utility function used to materialize a constant for a given attribute and type.
static MLIRContext * getContext(OpFoldResult val)
static bool isLegalToInline(InlinerInterface &interface, Region *src, Region *insertRegion, bool shouldCloneInlinedRegion, IRMapping &valueMapping)
Utility to check that all of the operations within 'src' can be inlined.
static Operation::operand_range getLowerBoundOperands(AffineForOp forOp)
static Operation::operand_range getUpperBoundOperands(AffineForOp forOp)
static void print(spirv::VerCapExtAttr triple, DialectAsmPrinter &printer)
static VectorType getVectorType(Type scalarTy, const VectorizationStrategy *strategy)
Returns the vector type resulting from applying the provided vectorization strategy on the scalar typ...
static int64_t getNumElements(ShapedType type)
RetTy walkPostOrder(AffineExpr expr)
Base type for affine expression.
AffineExpr floorDiv(uint64_t v) const
AffineExprKind getKind() const
Return the classification for this type.
int64_t getLargestKnownDivisor() const
Returns the greatest known integral divisor of this affine expression.
MLIRContext * getContext() const
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued.
AffineMap getSliceMap(unsigned start, unsigned length) const
Returns the map consisting of length expressions starting from start.
MLIRContext * getContext() const
bool isFunctionOfDim(unsigned position) const
Return true if any affine expression involves AffineDimExpr position.
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
AffineMap shiftDims(unsigned shift, unsigned offset=0) const
Replace dims[offset ...
unsigned getNumSymbols() const
unsigned getNumDims() const
ArrayRef< AffineExpr > getResults() const
bool isFunctionOfSymbol(unsigned position) const
Return true if any affine expression involves AffineSymbolExpr position.
unsigned getNumResults() const
AffineMap replaceDimsAndSymbols(ArrayRef< AffineExpr > dimReplacements, ArrayRef< AffineExpr > symReplacements, unsigned numResultDims, unsigned numResultSyms) const
This method substitutes any uses of dimensions and symbols (e.g.
unsigned getNumInputs() const
AffineMap shiftSymbols(unsigned shift, unsigned offset=0) const
Replace symbols[offset ...
static SmallVector< AffineMap, 4 > inferFromExprList(ArrayRef< ArrayRef< AffineExpr >> exprsList)
Returns a vector of AffineMaps; each with as many results as exprs.size(), as many dims as the larges...
AffineExpr getResult(unsigned idx) const
AffineMap replace(AffineExpr expr, AffineExpr replacement, unsigned numResultDims, unsigned numResultSyms) const
Sparse replace method.
static AffineMap getConstantMap(int64_t val, MLIRContext *context)
Returns a single constant result affine map.
AffineMap getSubMap(ArrayRef< unsigned > resultPos) const
Returns the map consisting of the resultPos subset.
LogicalResult constantFold(ArrayRef< Attribute > operandConstants, SmallVectorImpl< Attribute > &results, bool *hasPoison=nullptr) const
Folds the results of the application of an affine map on the provided operands to a constant if possi...
@ Paren
Parens surrounding zero or more operands.
@ OptionalSquare
Square brackets supporting zero or more ops, or nothing.
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
virtual ParseResult parseRParen()=0
Parse a ) token.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
ParseResult addTypeToList(Type type, SmallVectorImpl< Type > &result)
Add the specified type to the end of the specified type list and return success.
virtual ParseResult parseOptionalRParen()=0
Parse a ) token if present.
virtual ParseResult parseLess()=0
Parse a '<' token.
virtual ParseResult parseEqual()=0
Parse a = token.
virtual ParseResult parseColonType(Type &result)=0
Parse a colon followed by a type.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseGreater()=0
Parse a '>' token.
virtual ParseResult parseLParen()=0
Parse a ( token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
virtual ParseResult parseComma()=0
Parse a , token.
virtual ParseResult parseOptionalArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an optional arrow followed by a type list.
virtual ParseResult parseArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an arrow followed by a type list.
ParseResult parseKeyword(StringRef keyword)
Parse a given keyword.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
void printOptionalArrowTypeList(TypeRange &&types)
Print an optional arrow followed by a type list.
Attributes are known-constant values of operations.
Block represents an ordered list of Operations.
Operation * getTerminator()
Get the terminator operation of this block.
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
BlockArgListType getArguments()
This class is a general helper class for creating context-global objects like types,...
DenseI32ArrayAttr getDenseI32ArrayAttr(ArrayRef< int32_t > values)
IntegerAttr getIntegerAttr(Type type, int64_t value)
AffineMap getDimIdentityMap()
AffineMap getMultiDimIdentityMap(unsigned rank)
DenseIntElementsAttr getI32TensorAttr(ArrayRef< int32_t > values)
Tensor-typed DenseIntElementsAttr getters.
IntegerAttr getI64IntegerAttr(int64_t value)
IntegerType getIntegerType(unsigned width)
BoolAttr getBoolAttr(bool value)
AffineMap getEmptyAffineMap()
Returns a zero result affine map with no dimensions or symbols: () -> ().
AffineMap getConstantAffineMap(int64_t val)
Returns a single constant result affine map with 0 dimensions and 0 symbols.
MLIRContext * getContext() const
AffineMap getSymbolIdentityMap()
ArrayAttr getArrayAttr(ArrayRef< Attribute > value)
ArrayAttr getI64ArrayAttr(ArrayRef< int64_t > values)
An attribute that represents a reference to a dense integer vector or tensor object.
This is the interface that must be implemented by the dialects of operations to be inlined.
DialectInlinerInterface(Dialect *dialect)
This class provides support for representing a failure result, or a valid value of type T.
This is a utility class for mapping one set of IR entities to another.
auto lookup(T from) const
Lookup a mapped value within the map.
An integer set representing a conjunction of one or more affine equalities and inequalities.
unsigned getNumDims() const
static IntegerSet get(unsigned dimCount, unsigned symbolCount, ArrayRef< AffineExpr > constraints, ArrayRef< bool > eqFlags)
MLIRContext * getContext() const
unsigned getNumInputs() const
ArrayRef< AffineExpr > getConstraints() const
ArrayRef< bool > getEqFlags() const
Returns the equality bits, which specify whether each of the constraints is an equality or inequality...
unsigned getNumSymbols() const
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
MLIRContext is the top-level object for a collection of MLIR operations.
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
void pop_back()
Pop last element from list.
Attribute erase(StringAttr name)
Erase the attribute with the given name from the list.
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult parseRegion(Region ®ion, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region.
virtual ParseResult parseArgument(Argument &result, bool allowType=false, bool allowAttrs=false)=0
Parse a single argument with the following syntax:
ParseResult parseTrailingOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None)
Parse zero or more trailing SSA comma-separated trailing operand references with a specified surround...
virtual ParseResult parseArgumentList(SmallVectorImpl< Argument > &result, Delimiter delimiter=Delimiter::None, bool allowType=false, bool allowAttrs=false)=0
Parse zero or more arguments with a specified surrounding delimiter.
virtual ParseResult parseAffineMapOfSSAIds(SmallVectorImpl< UnresolvedOperand > &operands, Attribute &map, StringRef attrName, NamedAttrList &attrs, Delimiter delimiter=Delimiter::Square)=0
Parses an affine map attribute where dims and symbols are SSA operands.
ParseResult parseAssignmentList(SmallVectorImpl< Argument > &lhs, SmallVectorImpl< UnresolvedOperand > &rhs)
Parse a list of assignments of the form (x1 = y1, x2 = y2, ...)
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
ParseResult resolveOperands(Operands &&operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
virtual ParseResult parseAffineExprOfSSAIds(SmallVectorImpl< UnresolvedOperand > &dimOperands, SmallVectorImpl< UnresolvedOperand > &symbOperands, AffineExpr &expr)=0
Parses an affine expression where dims and symbols are SSA operands.
virtual ParseResult parseOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None, bool allowResultNumber=true, int requiredOperandCount=-1)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printAffineExprOfSSAIds(AffineExpr expr, ValueRange dimOperands, ValueRange symOperands)=0
Prints an affine expression of SSA ids with SSA id names used instead of dims and symbols.
virtual void printAffineMapOfSSAIds(AffineMapAttr mapAttr, ValueRange operands)=0
Prints an affine map of SSA ids, where SSA id names are used in place of dims/symbols.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
virtual void printRegionArgument(BlockArgument arg, ArrayRef< NamedAttribute > argAttrs={}, bool omitType=false)=0
Print a block argument in the usual format of: ssaName : type {attr1=42} loc("here") where location p...
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
Block::iterator getInsertionPoint() const
Returns the current insertion point of the builder.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Listener * getListener() const
Returns the current listener of this builder, or nullptr if this builder doesn't have a listener.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Block * getInsertionBlock() const
Return the block the current insertion point belongs to.
This class represents a single result from folding an operation.
This class represents an operand of an operation.
A trait of region holding operations that defines a new scope for polyhedral optimization purposes.
This class provides the API for ops that are known to be isolated from above.
A trait used to provide symbol table functionalities to a region operation.
This class implements the operand iterators for the Operation class.
Operation is the basic unit of execution within MLIR.
Value getOperand(unsigned idx)
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Attribute getAttr(StringAttr name)
Return the specified attribute if present, null otherwise.
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
MLIRContext * getContext()
Return the context this operation is associated with.
unsigned getNumOperands()
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
ArrayRef< NamedAttribute > getAttrs()
Return all of the attributes on this operation.
void setAttr(StringAttr name, Attribute value)
If the an attribute exists with the specified name, change it to the new value.
operand_range getOperands()
Returns an iterator on the underlying Value's.
Region * getParentRegion()
Returns the region to which the instruction belongs.
bool isProperAncestor(Operation *other)
Return true if this operation is a proper ancestor of the other operation.
operand_range::iterator operand_iterator
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
unsigned getNumResults()
Return the number of results held by this operation.
This class represents success/failure for parsing-like operations that find it important to chain tog...
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
This class represents a point being branched from in the methods of the RegionBranchOpInterface.
bool isParent() const
Returns true if branching from the parent op.
This class represents a successor of a region.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
void push_back(Block *block)
Operation * getParentOp()
Return the parent operation this region is attached to.
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
virtual void eraseBlock(Block *block)
This method erases all operations in a block.
virtual void replaceOp(Operation *op, ValueRange newValues)
This method replaces the results of the operation with the specified list of values.
void updateRootInPlace(Operation *root, CallableT &&callable)
This method is a utility wrapper around a root update of an operation.
void mergeBlocks(Block *source, Block *dest, ValueRange argValues=std::nullopt)
Inline the operations of block 'source' into the end of block 'dest'.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
virtual void finalizeRootUpdate(Operation *op)
This method is used to signal the end of a root update on the given operation.
void replaceUsesWithIf(Value from, Value to, function_ref< bool(OpOperand &)> functor)
Find uses of from and replace them with to if the functor returns true.
virtual void startRootUpdate(Operation *op)
This method is used to notify the rewriter that an in-place operation modification is about to happen...
virtual void inlineBlockBefore(Block *source, Block *dest, Block::iterator before, ValueRange argValues=std::nullopt)
Inline the operations of block 'source' into block 'dest' before the given position.
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replaces the result op with a new op that is created without verification.
This class represents a specific instance of an effect.
static DerivedEffect * get()
Returns a unique instance for the derived effect class.
static DefaultResource * get()
Returns a unique instance for the given effect class.
std::vector< SmallVector< int64_t, 8 > > operandExprStack
static Operation * lookupSymbolIn(Operation *op, StringAttr symbol)
Returns the operation registered with the given symbol name with the regions of 'symbolTableOp'.
This class provides an abstraction over the various different ranges of value types.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
AffineBound represents a lower or upper bound in the for operation.
AffineDmaStartOp starts a non-blocking DMA operation that transfers data from a source memref to a de...
AffineDmaWaitOp blocks until the completion of a DMA operation associated with the tag element 'tag[i...
An AffineValueMap is an affine map plus its ML value operands and results for analysis purposes.
LogicalResult canonicalize()
Attempts to canonicalize the map and operands.
ArrayRef< Value > getOperands() const
AffineExpr getResult(unsigned i)
AffineMap getAffineMap() const
unsigned getNumResults() const
Operation * getOwner() const
Return the owner of this operand.
constexpr auto RecursivelySpeculatable
Speculatability
This enum is returned from the getSpeculatability method in the ConditionallySpeculatable op interfac...
constexpr auto NotSpeculatable
void buildAffineLoopNest(OpBuilder &builder, Location loc, ArrayRef< int64_t > lbs, ArrayRef< int64_t > ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn=nullptr)
Builds a perfect nest of affine.for loops, i.e., each loop except the innermost one contains only ano...
void fullyComposeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Given an affine map map and its input operands, this method composes into map, maps of AffineApplyOps...
void extractForInductionVars(ArrayRef< AffineForOp > forInsts, SmallVectorImpl< Value > *ivs)
Extracts the induction variables from a list of AffineForOps and places them in the output argument i...
bool isValidDim(Value value)
Returns true if the given Value can be used as a dimension id in the region of the closest surroundin...
SmallVector< OpFoldResult > makeComposedFoldedMultiResultAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Variant of makeComposedFoldedAffineApply suitable for multi-result maps.
bool isAffineInductionVar(Value val)
Returns true if the provided value is the induction variable of an AffineForOp or AffineParallelOp.
AffineForOp getForInductionVarOwner(Value val)
Returns the loop parent of an induction variable.
AffineApplyOp makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Returns a composed AffineApplyOp by composing map and operands with other AffineApplyOps supplying th...
void canonicalizeMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Modifies both map and operands in-place so as to:
OpFoldResult makeComposedFoldedAffineMax(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineMinOp that computes a maximum across the results of applying map to operands,...
bool isAffineForInductionVar(Value val)
Returns true if the provided value is the induction variable of an AffineForOp.
OpFoldResult makeComposedFoldedAffineMin(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineMinOp that computes a minimum across the results of applying map to operands,...
bool isTopLevelValue(Value value)
A utility function to check if a value is defined at the top level of an op with trait AffineScope or...
void canonicalizeSetAndOperands(IntegerSet *set, SmallVectorImpl< Value > *operands)
Canonicalizes an integer set the same way canonicalizeMapAndOperands does for affine maps.
void extractInductionVars(ArrayRef< Operation * > affineOps, SmallVectorImpl< Value > &ivs)
Extracts the induction variables from a list of either AffineForOp or AffineParallelOp and places the...
bool isValidSymbol(Value value)
Returns true if the given value can be used as a symbol in the region of the closest surrounding op t...
OpFoldResult makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineApplyOp that applies map to operands after composing the map with the maps of any...
AffineParallelOp getAffineParallelInductionVarOwner(Value val)
Returns true if the provided value is among the induction variables of an AffineParallelOp.
Region * getAffineScope(Operation *op)
Returns the closest region enclosing op that is held by an operation with trait AffineScope; nullptr ...
ParseResult parseDimAndSymbolList(OpAsmParser &parser, SmallVectorImpl< Value > &operands, unsigned &numDims)
Parses dimension and symbol list.
bool isAffineParallelInductionVar(Value val)
Returns true if val is the induction variable of an AffineParallelOp.
AffineMinOp makeComposedAffineMin(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Returns an AffineMinOp obtained by composing map and operands with AffineApplyOps supplying those ope...
BaseMemRefType getMemRefType(Value value, const BufferizationOptions &options, MemRefLayoutAttrInterface layout={}, Attribute memorySpace=nullptr)
Return a MemRefType to which the type of the given value can be bufferized.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
LogicalResult foldMemRefCast(Operation *op, Value inner=nullptr)
This is a common utility used for patterns of the form "someop(memref.cast) -> someop".
LLVM_ATTRIBUTE_ALWAYS_INLINE MPInt gcd(const MPInt &a, const MPInt &b)
QueryRef parse(llvm::StringRef line, const QuerySession &qs)
Include the generated interface declarations.
AffineMap simplifyAffineMap(AffineMap map)
Simplifies an affine map by simplifying its underlying AffineExpr results.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
AffineMap removeDuplicateExprs(AffineMap map)
Returns a map with the same dimension and symbol count as map, but whose results are the unique affin...
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
std::optional< int64_t > getBoundForAffineExpr(AffineExpr expr, unsigned numDims, unsigned numSymbols, ArrayRef< std::optional< int64_t >> constLowerBounds, ArrayRef< std::optional< int64_t >> constUpperBounds, bool isUpper)
Get a lower or upper (depending on isUpper) bound for expr while using the constant lower and upper b...
int64_t floorDiv(int64_t lhs, int64_t rhs)
Returns the result of MLIR's floordiv operation on constants.
int64_t ceilDiv(int64_t lhs, int64_t rhs)
Returns the result of MLIR's ceildiv operation on constants.
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
@ CeilDiv
RHS of ceildiv is always a constant or a symbolic expression.
@ Mod
RHS of mod is always a constant or a symbolic expression with a positive value.
@ DimId
Dimensional identifier.
@ FloorDiv
RHS of floordiv is always a constant or a symbolic expression.
@ SymbolId
Symbolic identifier.
AffineExpr getAffineBinaryOpExpr(AffineExprKind kind, AffineExpr lhs, AffineExpr rhs)
std::function< SmallVector< Value >(OpBuilder &b, Location loc, ArrayRef< BlockArgument > newBbArgs)> NewYieldValuesFn
A function that returns the additional yielded values during replaceWithAdditionalYields.
AffineExpr getAffineConstantExpr(int64_t constant, MLIRContext *context)
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool isStrided(MemRefType t)
Return "true" if the layout for t is compatible with strided semantics.
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
AffineExpr getAffineDimExpr(unsigned position, MLIRContext *context)
These free functions allow clients of the API to not use classes in detail.
LogicalResult verify(Operation *op, bool verifyRecursively=true)
Perform (potentially expensive) checks of invariants, used to detect compiler bugs,...
AffineMap foldAttributesIntoMap(Builder &b, AffineMap map, ArrayRef< OpFoldResult > operands, SmallVector< Value > &remainingValues)
Fold all attributes among the given operands into the affine map.
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
AffineExpr getAffineSymbolExpr(unsigned position, MLIRContext *context)
Canonicalize the affine map result expression order of an affine min/max operation.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Remove duplicated expressions in affine min/max ops.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Merge an affine min/max op to its consumers if its consumer is also an affine min/max op.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
This class represents an efficient way to signal success or failure.
This is the representation of an operand reference.
This class represents a listener that may be used to hook into various actions within an OpBuilder.
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
This represents an operation in an abstracted form, suitable for use with the builder APIs.
T & getOrAddProperties()
Get (or create) a properties of the provided type to be set on the operation on creation.
SmallVector< Value, 4 > operands
void addOperands(ValueRange newOperands)
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
void addTypes(ArrayRef< Type > newTypes)
SmallVector< std::unique_ptr< Region >, 1 > regions
Regions that the op will hold.
SmallVector< Type, 4 > types
Types of the results of this operation.
Region * addRegion()
Create a region that should be attached to the operation.