21 #include "llvm/ADT/ScopeExit.h"
22 #include "llvm/ADT/SmallBitVector.h"
23 #include "llvm/ADT/SmallVectorExtras.h"
24 #include "llvm/ADT/TypeSwitch.h"
25 #include "llvm/Support/Debug.h"
32 #define DEBUG_TYPE "affine-ops"
34 #include "mlir/Dialect/Affine/IR/AffineOpsDialect.cpp.inc"
41 if (
auto arg = llvm::dyn_cast<BlockArgument>(value))
42 return arg.getParentRegion() == region;
65 if (llvm::isa<BlockArgument>(value))
66 return legalityCheck(mapping.
lookup(value), dest);
73 bool isDimLikeOp = isa<ShapedDimOpInterface>(value.
getDefiningOp());
84 return llvm::all_of(values, [&](
Value v) {
91 template <
typename OpTy>
94 static_assert(llvm::is_one_of<OpTy, AffineReadOpInterface,
95 AffineWriteOpInterface>::value,
96 "only ops with affine read/write interface are supported");
103 dimOperands, src, dest, mapping,
107 symbolOperands, src, dest, mapping,
124 op.getMapOperands(), src, dest, mapping,
129 op.getMapOperands(), src, dest, mapping,
156 if (!isa<AffineParallelOp, AffineForOp, AffineIfOp>(destOp))
161 if (!llvm::hasSingleElement(*src))
169 if (
auto iface = dyn_cast<MemoryEffectOpInterface>(op)) {
170 if (iface.hasNoEffect())
178 .Case<AffineApplyOp, AffineReadOpInterface,
179 AffineWriteOpInterface>([&](
auto op) {
204 isa<AffineForOp, AffineParallelOp, AffineIfOp>(parentOp);
208 bool shouldAnalyzeRecursively(
Operation *op)
const final {
return true; }
216 void AffineDialect::initialize() {
219 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc"
221 addInterfaces<AffineInlinerInterface>();
229 return arith::ConstantOp::materialize(builder, value, type, loc);
237 if (
auto arg = llvm::dyn_cast<BlockArgument>(value)) {
253 while (
auto *parentOp = curOp->getParentOp()) {
276 auto *parentOp = llvm::cast<BlockArgument>(value).getOwner()->getParentOp();
278 isa<AffineForOp, AffineParallelOp>(parentOp));
299 auto *parentOp = llvm::cast<BlockArgument>(value).getOwner()->
getParentOp();
300 return isa<AffineForOp, AffineParallelOp>(parentOp);
304 if (
auto applyOp = dyn_cast<AffineApplyOp>(op))
305 return applyOp.isValidDim(region);
308 if (
auto dimOp = dyn_cast<ShapedDimOpInterface>(op))
316 template <
typename AnyMemRefDefOp>
319 auto memRefType = memrefDefOp.getType();
321 if (!memRefType.isDynamicDim(index))
324 unsigned dynamicDimPos = memRefType.getDynamicDimIndex(index);
325 return isValidSymbol(*(memrefDefOp.getDynamicSizes().begin() + dynamicDimPos),
337 if (llvm::isa<BlockArgument>(dimOp.getShapedValue()))
345 if (!index.has_value())
348 int64_t i = index.value();
350 .Case<memref::ViewOp, memref::SubViewOp, memref::AllocOp>(
352 .Default([](
Operation *) {
return false; });
418 if (
auto applyOp = dyn_cast<AffineApplyOp>(defOp))
419 return applyOp.isValidSymbol(region);
422 if (
auto dimOp = dyn_cast<ShapedDimOpInterface>(defOp))
446 printer <<
'(' << operands.take_front(numDims) <<
')';
447 if (operands.size() > numDims)
448 printer <<
'[' << operands.drop_front(numDims) <<
']';
458 numDims = opInfos.size();
472 template <
typename OpTy>
477 for (
auto operand : operands) {
478 if (opIt++ < numDims) {
480 return op.
emitOpError(
"operand cannot be used as a dimension id");
482 return op.
emitOpError(
"operand cannot be used as a symbol");
493 return AffineValueMap(getAffineMap(), getOperands(), getResult());
500 AffineMapAttr mapAttr;
506 auto map = mapAttr.getValue();
508 if (map.getNumDims() != numDims ||
509 numDims + map.getNumSymbols() != result.
operands.size()) {
511 "dimension or symbol index mismatch");
514 result.
types.append(map.getNumResults(), indexTy);
519 p <<
" " << getMapAttr();
521 getAffineMap().getNumDims(), p);
532 "operand count and affine map dimension and symbol count must match");
536 return emitOpError(
"mapping must produce one value");
544 return llvm::all_of(getOperands(),
552 return llvm::all_of(getOperands(),
559 return llvm::all_of(getOperands(),
566 return llvm::all_of(getOperands(), [&](
Value operand) {
572 auto map = getAffineMap();
575 auto expr = map.getResult(0);
577 return getOperand(dim.getPosition());
579 return getOperand(map.getNumDims() + sym.getPosition());
583 if (
failed(map.constantFold(adaptor.getMapOperands(), result)))
610 Value operand = operands[dimExpr.getPosition()];
611 int64_t operandDivisor = 1;
615 if (forOp.hasConstantLowerBound() && forOp.getConstantLowerBound() == 0) {
616 operandDivisor = forOp.getStep();
618 uint64_t lbLargestKnownDivisor =
619 forOp.getLowerBoundMap().getLargestKnownDivisorOfMapExprs();
620 operandDivisor =
std::gcd(lbLargestKnownDivisor, forOp.getStep());
623 return operandDivisor;
631 int64_t constVal = constExpr.getValue();
632 return constVal >= 0 && constVal < k;
637 Value operand = operands[dimExpr.getPosition()];
641 if (forOp.hasConstantLowerBound() && forOp.getConstantLowerBound() >= 0 &&
642 forOp.hasConstantUpperBound() && forOp.getConstantUpperBound() <= k) {
666 quotientTimesDiv = llhs;
672 quotientTimesDiv = rlhs;
682 if (forOp && forOp.hasConstantLowerBound())
683 return forOp.getConstantLowerBound();
690 if (!forOp || !forOp.hasConstantUpperBound())
695 if (forOp.hasConstantLowerBound()) {
696 return forOp.getConstantUpperBound() - 1 -
697 (forOp.getConstantUpperBound() - forOp.getConstantLowerBound() - 1) %
700 return forOp.getConstantUpperBound() - 1;
711 static std::optional<int64_t>
713 ArrayRef<std::optional<int64_t>> constLowerBounds,
714 ArrayRef<std::optional<int64_t>> constUpperBounds,
722 if (!rhsConst || rhsConst.getValue() < 1)
725 constLowerBounds, constUpperBounds, isUpper);
732 if (rhsConst && rhsConst.getValue() >= 1) {
735 constLowerBounds, constUpperBounds, isUpper);
747 if (rhsConst && rhsConst.getValue() >= 1) {
748 int64_t rhsConstVal = rhsConst.
getValue();
750 constLowerBounds, constUpperBounds,
753 constLowerBounds, constUpperBounds, isUpper);
756 return isUpper ?
mod(*ub, rhsConstVal) :
mod(*lb, rhsConstVal);
757 return isUpper ? rhsConstVal - 1 : 0;
772 for (
unsigned i = 0, e = numDims + numSymbols; i < e; ++i) {
773 if (flattenedExpr[i] > 0) {
774 auto &constBound = isUpper ? constUpperBounds[i] : constLowerBounds[i];
777 bound += *constBound * flattenedExpr[i];
778 }
else if (flattenedExpr[i] < 0) {
779 auto &constBound = isUpper ? constLowerBounds[i] : constUpperBounds[i];
782 bound += *constBound * flattenedExpr[i];
786 bound += flattenedExpr.back();
798 constLowerBounds.reserve(operands.size());
799 constUpperBounds.reserve(operands.size());
800 for (
Value operand : operands) {
806 return constExpr.getValue();
821 constLowerBounds.reserve(operands.size());
822 constUpperBounds.reserve(operands.size());
823 for (
Value operand : operands) {
828 std::optional<int64_t> lowerBound;
830 lowerBound = constExpr.getValue();
832 lowerBound =
getBoundForExpr(expr, numDims, numSymbols, constLowerBounds,
863 lhs = binExpr.getLHS();
864 rhs = binExpr.getRHS();
869 int64_t rhsConstVal = rhsConst.
getValue();
871 if (rhsConstVal <= 0)
876 std::optional<int64_t> lhsLbConst =
878 std::optional<int64_t> lhsUbConst =
880 if (lhsLbConst && lhsUbConst) {
881 int64_t lhsLbConstVal = *lhsLbConst;
882 int64_t lhsUbConstVal = *lhsUbConst;
886 floorDiv(lhsLbConstVal, rhsConstVal) ==
887 floorDiv(lhsUbConstVal, rhsConstVal)) {
895 ceilDiv(lhsLbConstVal, rhsConstVal) ==
896 ceilDiv(lhsUbConstVal, rhsConstVal)) {
903 lhsLbConstVal < rhsConstVal && lhsUbConstVal < rhsConstVal) {
915 if (
isQTimesDPlusR(lhs, operands, divisor, quotientTimesDiv, rem)) {
916 if (rhsConstVal % divisor == 0 &&
918 expr = quotientTimesDiv.
floorDiv(rhsConst);
919 }
else if (divisor % rhsConstVal == 0 &&
921 expr = rem % rhsConst;
947 if (operands.empty())
953 constLowerBounds.reserve(operands.size());
954 constUpperBounds.reserve(operands.size());
955 for (
Value operand : operands) {
970 lowerBounds.push_back(constExpr.getValue());
971 upperBounds.push_back(constExpr.getValue());
975 constLowerBounds, constUpperBounds,
979 constLowerBounds, constUpperBounds,
988 unsigned i = exprEn.index();
990 if (lowerBounds[i] && upperBounds[i] && *lowerBounds[i] == *upperBounds[i])
995 if (!upperBounds[i]) {
996 irredundantExprs.push_back(e);
1002 auto otherLowerBound = en.value();
1003 unsigned pos = en.index();
1004 if (pos == i || !otherLowerBound)
1006 if (*otherLowerBound > *upperBounds[i])
1008 if (*otherLowerBound < *upperBounds[i])
1013 if (upperBounds[pos] && lowerBounds[i] &&
1014 lowerBounds[i] == upperBounds[i] &&
1015 otherLowerBound == *upperBounds[pos] && i < pos)
1019 irredundantExprs.push_back(e);
1021 if (!lowerBounds[i]) {
1022 irredundantExprs.push_back(e);
1027 auto otherUpperBound = en.value();
1028 unsigned pos = en.index();
1029 if (pos == i || !otherUpperBound)
1031 if (*otherUpperBound < *lowerBounds[i])
1033 if (*otherUpperBound > *lowerBounds[i])
1035 if (lowerBounds[pos] && upperBounds[i] &&
1036 lowerBounds[i] == upperBounds[i] &&
1037 otherUpperBound == lowerBounds[pos] && i < pos)
1041 irredundantExprs.push_back(e);
1053 static void LLVM_ATTRIBUTE_UNUSED
1055 assert(map.
getNumInputs() == operands.size() &&
"invalid operands for map");
1061 newResults.push_back(expr);
1078 unsigned dimOrSymbolPosition,
1082 bool isDimReplacement = (dimOrSymbolPosition < dims.size());
1083 unsigned pos = isDimReplacement ? dimOrSymbolPosition
1084 : dimOrSymbolPosition - dims.size();
1085 Value &v = isDimReplacement ? dims[pos] : syms[pos];
1098 AffineMap composeMap = affineApply.getAffineMap();
1099 assert(composeMap.
getNumResults() == 1 &&
"affine.apply with >1 results");
1101 affineApply.getMapOperands().end());
1115 dims.append(composeDims.begin(), composeDims.end());
1116 syms.append(composeSyms.begin(), composeSyms.end());
1117 *map = map->
replace(toReplace, replacementExpr, dims.size(), syms.size());
1145 bool changed =
false;
1146 for (
unsigned pos = 0; pos != dims.size() + syms.size(); ++pos)
1158 unsigned nDims = 0, nSyms = 0;
1160 dimReplacements.reserve(dims.size());
1161 symReplacements.reserve(syms.size());
1162 for (
auto *container : {&dims, &syms}) {
1163 bool isDim = (container == &dims);
1164 auto &repls = isDim ? dimReplacements : symReplacements;
1166 Value v = en.value();
1170 "map is function of unexpected expr@pos");
1176 operands->push_back(v);
1189 while (llvm::any_of(*operands, [](
Value v) {
1209 auto listenerResetter =
1210 llvm::make_scope_exit([listener, &b] { b.
setListener(listener); });
1212 actualValues.reserve(values.size());
1215 if (
auto value = llvm::dyn_cast_if_present<Value>(ofr)) {
1216 actualValues.push_back(value);
1223 constants.push_back(dialect->materializeConstant(
1227 actualValues.push_back(constants.back()->getResult(0));
1238 template <
typename OpTy,
typename... Args>
1239 static std::enable_if_t<OpTy::template hasTrait<OpTrait::OneResult>(),
1242 Args &&...leadingArguments) {
1247 constantOperands.reserve(operands.size());
1248 for (
Value operand : operands) {
1251 constantOperands.push_back(attr);
1253 constantOperands.push_back(
nullptr);
1267 auto listenerResetter =
1268 llvm::make_scope_exit([listener, &b] { b.
setListener(listener); });
1270 b.
create<OpTy>(loc, std::forward<Args>(leadingArguments)..., operands);
1273 !foldResults.empty()) {
1275 return foldResults.front();
1291 assert(normalizedMap);
1292 return b.
create<AffineApplyOp>(loc, normalizedMap, normalizedOperands);
1312 for (
unsigned i : llvm::seq<unsigned>(0, map.
getNumResults())) {
1319 llvm::append_range(dims,
1321 llvm::append_range(symbols,
1328 operands = llvm::to_vector(llvm::concat<Value>(dims, symbols));
1337 assert(map.
getNumResults() == 1 &&
"building affine.apply with !=1 result");
1343 OpFoldResult result = createOrFold<AffineApplyOp>(b, loc, actualValues, map);
1365 return llvm::map_to_vector(llvm::seq<unsigned>(0, map.
getNumResults()),
1367 return makeComposedFoldedAffineApply(
1368 b, loc, map.getSubMap({i}), operands);
1379 template <
typename OpTy>
1388 createOrFold<OpTy>(b, loc, actualValues, b.
getIndexType(), map);
1401 return makeComposedFoldedMinMax<AffineMinOp>(b, loc, map, operands);
1408 return makeComposedFoldedMinMax<AffineMaxOp>(b, loc, map, operands);
1419 return b.
createOrFold<AffineApplyOp>(loc, map, operands);
1440 template <
class MapOrSet>
1443 if (!mapOrSet || operands->empty())
1446 assert(mapOrSet->getNumInputs() == operands->size() &&
1447 "map/set inputs must match number of operands");
1449 auto *context = mapOrSet->getContext();
1451 resultOperands.reserve(operands->size());
1453 remappedSymbols.reserve(operands->size());
1454 unsigned nextDim = 0;
1455 unsigned nextSym = 0;
1456 unsigned oldNumSyms = mapOrSet->getNumSymbols();
1458 for (
unsigned i = 0, e = mapOrSet->getNumInputs(); i != e; ++i) {
1459 if (i < mapOrSet->getNumDims()) {
1463 remappedSymbols.push_back((*operands)[i]);
1466 resultOperands.push_back((*operands)[i]);
1469 resultOperands.push_back((*operands)[i]);
1473 resultOperands.append(remappedSymbols.begin(), remappedSymbols.end());
1474 *operands = resultOperands;
1475 *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, {}, nextDim,
1476 oldNumSyms + nextSym);
1478 assert(mapOrSet->getNumInputs() == operands->size() &&
1479 "map/set inputs must match number of operands");
1483 template <
class MapOrSet>
1486 static_assert(llvm::is_one_of<MapOrSet, AffineMap, IntegerSet>::value,
1487 "Argument must be either of AffineMap or IntegerSet type");
1489 if (!mapOrSet || operands->empty())
1492 assert(mapOrSet->getNumInputs() == operands->size() &&
1493 "map/set inputs must match number of operands");
1495 canonicalizePromotedSymbols<MapOrSet>(mapOrSet, operands);
1498 llvm::SmallBitVector usedDims(mapOrSet->getNumDims());
1499 llvm::SmallBitVector usedSyms(mapOrSet->getNumSymbols());
1502 usedDims[dimExpr.getPosition()] =
true;
1504 usedSyms[symExpr.getPosition()] =
true;
1507 auto *context = mapOrSet->getContext();
1510 resultOperands.reserve(operands->size());
1512 llvm::SmallDenseMap<Value, AffineExpr, 8> seenDims;
1514 unsigned nextDim = 0;
1515 for (
unsigned i = 0, e = mapOrSet->getNumDims(); i != e; ++i) {
1518 auto it = seenDims.find((*operands)[i]);
1519 if (it == seenDims.end()) {
1521 resultOperands.push_back((*operands)[i]);
1522 seenDims.insert(std::make_pair((*operands)[i], dimRemapping[i]));
1524 dimRemapping[i] = it->second;
1528 llvm::SmallDenseMap<Value, AffineExpr, 8> seenSymbols;
1530 unsigned nextSym = 0;
1531 for (
unsigned i = 0, e = mapOrSet->getNumSymbols(); i != e; ++i) {
1537 IntegerAttr operandCst;
1538 if (
matchPattern((*operands)[i + mapOrSet->getNumDims()],
1545 auto it = seenSymbols.find((*operands)[i + mapOrSet->getNumDims()]);
1546 if (it == seenSymbols.end()) {
1548 resultOperands.push_back((*operands)[i + mapOrSet->getNumDims()]);
1549 seenSymbols.insert(std::make_pair((*operands)[i + mapOrSet->getNumDims()],
1552 symRemapping[i] = it->second;
1555 *mapOrSet = mapOrSet->replaceDimsAndSymbols(dimRemapping, symRemapping,
1557 *operands = resultOperands;
1562 canonicalizeMapOrSetAndOperands<AffineMap>(map, operands);
1567 canonicalizeMapOrSetAndOperands<IntegerSet>(set, operands);
1574 template <
typename AffineOpTy>
1586 llvm::is_one_of<AffineOpTy, AffineLoadOp, AffinePrefetchOp,
1587 AffineStoreOp, AffineApplyOp, AffineMinOp, AffineMaxOp,
1588 AffineVectorStoreOp, AffineVectorLoadOp>::value,
1589 "affine load/store/vectorstore/vectorload/apply/prefetch/min/max op "
1591 auto map = affineOp.getAffineMap();
1593 auto oldOperands = affineOp.getMapOperands();
1598 if (map == oldMap && std::equal(oldOperands.begin(), oldOperands.end(),
1599 resultOperands.begin()))
1602 replaceAffineOp(rewriter, affineOp, map, resultOperands);
1610 void SimplifyAffineOp<AffineLoadOp>::replaceAffineOp(
1617 void SimplifyAffineOp<AffinePrefetchOp>::replaceAffineOp(
1621 prefetch, prefetch.getMemref(), map, mapOperands,
1622 prefetch.getLocalityHint(), prefetch.getIsWrite(),
1623 prefetch.getIsDataCache());
1626 void SimplifyAffineOp<AffineStoreOp>::replaceAffineOp(
1630 store, store.getValueToStore(), store.getMemRef(), map, mapOperands);
1633 void SimplifyAffineOp<AffineVectorLoadOp>::replaceAffineOp(
1637 vectorload, vectorload.getVectorType(), vectorload.getMemRef(), map,
1641 void SimplifyAffineOp<AffineVectorStoreOp>::replaceAffineOp(
1645 vectorstore, vectorstore.getValueToStore(), vectorstore.getMemRef(), map,
1650 template <
typename AffineOpTy>
1651 void SimplifyAffineOp<AffineOpTy>::replaceAffineOp(
1660 results.
add<SimplifyAffineOp<AffineApplyOp>>(context);
1691 p <<
" " << getSrcMemRef() <<
'[';
1693 p <<
"], " << getDstMemRef() <<
'[';
1695 p <<
"], " << getTagMemRef() <<
'[';
1699 p <<
", " << getStride();
1700 p <<
", " << getNumElementsPerStride();
1702 p <<
" : " << getSrcMemRefType() <<
", " << getDstMemRefType() <<
", "
1703 << getTagMemRefType();
1715 AffineMapAttr srcMapAttr;
1718 AffineMapAttr dstMapAttr;
1721 AffineMapAttr tagMapAttr;
1736 getSrcMapAttrStrName(),
1740 getDstMapAttrStrName(),
1744 getTagMapAttrStrName(),
1753 if (!strideInfo.empty() && strideInfo.size() != 2) {
1755 "expected two stride related operands");
1757 bool isStrided = strideInfo.size() == 2;
1762 if (types.size() != 3)
1780 if (srcMapOperands.size() != srcMapAttr.getValue().getNumInputs() ||
1781 dstMapOperands.size() != dstMapAttr.getValue().getNumInputs() ||
1782 tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1784 "memref operand count not equal to map.numInputs");
1789 if (!llvm::isa<MemRefType>(getOperand(getSrcMemRefOperandIndex()).getType()))
1790 return emitOpError(
"expected DMA source to be of memref type");
1791 if (!llvm::isa<MemRefType>(getOperand(getDstMemRefOperandIndex()).getType()))
1792 return emitOpError(
"expected DMA destination to be of memref type");
1793 if (!llvm::isa<MemRefType>(getOperand(getTagMemRefOperandIndex()).getType()))
1794 return emitOpError(
"expected DMA tag to be of memref type");
1796 unsigned numInputsAllMaps = getSrcMap().getNumInputs() +
1797 getDstMap().getNumInputs() +
1798 getTagMap().getNumInputs();
1799 if (getNumOperands() != numInputsAllMaps + 3 + 1 &&
1800 getNumOperands() != numInputsAllMaps + 3 + 1 + 2) {
1801 return emitOpError(
"incorrect number of operands");
1805 for (
auto idx : getSrcIndices()) {
1806 if (!idx.getType().isIndex())
1807 return emitOpError(
"src index to dma_start must have 'index' type");
1809 return emitOpError(
"src index must be a dimension or symbol identifier");
1811 for (
auto idx : getDstIndices()) {
1812 if (!idx.getType().isIndex())
1813 return emitOpError(
"dst index to dma_start must have 'index' type");
1815 return emitOpError(
"dst index must be a dimension or symbol identifier");
1817 for (
auto idx : getTagIndices()) {
1818 if (!idx.getType().isIndex())
1819 return emitOpError(
"tag index to dma_start must have 'index' type");
1821 return emitOpError(
"tag index must be a dimension or symbol identifier");
1832 void AffineDmaStartOp::getEffects(
1858 p <<
" " << getTagMemRef() <<
'[';
1863 p <<
" : " << getTagMemRef().getType();
1874 AffineMapAttr tagMapAttr;
1883 getTagMapAttrStrName(),
1892 if (!llvm::isa<MemRefType>(type))
1894 "expected tag to be of memref type");
1896 if (tagMapOperands.size() != tagMapAttr.getValue().getNumInputs())
1898 "tag memref operand count != to map.numInputs");
1903 if (!llvm::isa<MemRefType>(getOperand(0).getType()))
1904 return emitOpError(
"expected DMA tag to be of memref type");
1906 for (
auto idx : getTagIndices()) {
1907 if (!idx.getType().isIndex())
1908 return emitOpError(
"index to dma_wait must have 'index' type");
1910 return emitOpError(
"index must be a dimension or symbol identifier");
1921 void AffineDmaWaitOp::getEffects(
1937 ValueRange iterArgs, BodyBuilderFn bodyBuilder) {
1938 assert(((!lbMap && lbOperands.empty()) ||
1940 "lower bound operand count does not match the affine map");
1941 assert(((!ubMap && ubOperands.empty()) ||
1943 "upper bound operand count does not match the affine map");
1944 assert(step > 0 &&
"step has to be a positive integer constant");
1946 for (
Value val : iterArgs)
1967 Value inductionVar =
1969 for (
Value val : iterArgs)
1970 bodyBlock.
addArgument(val.getType(), val.getLoc());
1975 if (iterArgs.empty() && !bodyBuilder) {
1976 ensureTerminator(*bodyRegion, builder, result.
location);
1977 }
else if (bodyBuilder) {
1980 bodyBuilder(builder, result.
location, inductionVar,
1986 int64_t ub, int64_t step,
ValueRange iterArgs,
1987 BodyBuilderFn bodyBuilder) {
1990 return build(builder, result, {}, lbMap, {}, ubMap, step, iterArgs,
1997 auto *body = getBody();
1998 if (body->getNumArguments() == 0 || !body->getArgument(0).getType().isIndex())
1999 return emitOpError(
"expected body to have a single index argument for the "
2000 "induction variable");
2004 if (getLowerBoundMap().getNumInputs() > 0)
2006 getLowerBoundMap().getNumDims())))
2009 if (getUpperBoundMap().getNumInputs() > 0)
2011 getUpperBoundMap().getNumDims())))
2014 unsigned opNumResults = getNumResults();
2015 if (opNumResults == 0)
2021 if (getNumIterOperands() != opNumResults)
2023 "mismatch between the number of loop-carried values and results");
2024 if (getNumRegionIterArgs() != opNumResults)
2026 "mismatch between the number of basic block args and results");
2036 bool failedToParsedMinMax =
2040 auto boundAttrStrName = isLower ? AffineForOp::getLowerBoundAttrStrName()
2041 : AffineForOp::getUpperBoundAttrStrName();
2048 if (!boundOpInfos.empty()) {
2050 if (boundOpInfos.size() > 1)
2052 "expected only one loop bound operand");
2077 if (
auto affineMapAttr = llvm::dyn_cast<AffineMapAttr>(boundAttr)) {
2078 unsigned currentNumOperands = result.
operands.size();
2083 auto map = affineMapAttr.getValue();
2087 "dim operand count and affine map dim count must match");
2089 unsigned numDimAndSymbolOperands =
2090 result.
operands.size() - currentNumOperands;
2091 if (numDims + map.
getNumSymbols() != numDimAndSymbolOperands)
2094 "symbol operand count and affine map symbol count must match");
2100 return p.
emitError(attrLoc,
"lower loop bound affine map with "
2101 "multiple results requires 'max' prefix");
2103 return p.
emitError(attrLoc,
"upper loop bound affine map with multiple "
2104 "results requires 'min' prefix");
2110 if (
auto integerAttr = llvm::dyn_cast<IntegerAttr>(boundAttr)) {
2120 "expected valid affine map representation for loop bounds");
2140 AffineForOp::getStepAttrStrName(),
2144 IntegerAttr stepAttr;
2146 AffineForOp::getStepAttrStrName().data(),
2150 if (stepAttr.getValue().isNegative())
2153 "expected step to be representable as a positive signed integer");
2161 regionArgs.push_back(inductionVariable);
2169 for (
auto argOperandType :
2170 llvm::zip(llvm::drop_begin(regionArgs), operands, result.
types)) {
2171 Type type = std::get<2>(argOperandType);
2172 std::get<0>(argOperandType).type = type;
2181 if (regionArgs.size() != result.
types.size() + 1)
2184 "mismatch between the number of loop-carried values and results");
2188 AffineForOp::ensureTerminator(*body, builder, result.
location);
2211 p << constExpr.getValue();
2235 unsigned AffineForOp::getNumIterOperands() {
2236 AffineMap lbMap = getLowerBoundMapAttr().getValue();
2237 AffineMap ubMap = getUpperBoundMapAttr().getValue();
2252 p <<
" step " << getStep();
2254 bool printBlockTerminators =
false;
2255 if (getNumIterOperands() > 0) {
2257 auto regionArgs = getRegionIterArgs();
2258 auto operands = getIterOperands();
2260 llvm::interleaveComma(llvm::zip(regionArgs, operands), p, [&](
auto it) {
2261 p << std::get<0>(it) <<
" = " << std::get<1>(it);
2263 p <<
") -> (" << getResultTypes() <<
")";
2264 printBlockTerminators =
true;
2269 printBlockTerminators);
2271 {getLowerBoundAttrStrName(),
2272 getUpperBoundAttrStrName(),
2273 getStepAttrStrName()});
2278 auto foldLowerOrUpperBound = [&forOp](
bool lower) {
2282 auto boundOperands =
2283 lower ? forOp.getLowerBoundOperands() : forOp.getUpperBoundOperands();
2284 for (
auto operand : boundOperands) {
2287 operandConstants.push_back(operandCst);
2291 lower ? forOp.getLowerBoundMap() : forOp.getUpperBoundMap();
2293 "bound maps should have at least one result");
2299 assert(!foldedResults.empty() &&
"bounds should have at least one result");
2300 auto maxOrMin = llvm::cast<IntegerAttr>(foldedResults[0]).getValue();
2301 for (
unsigned i = 1, e = foldedResults.size(); i < e; i++) {
2302 auto foldedResult = llvm::cast<IntegerAttr>(foldedResults[i]).getValue();
2303 maxOrMin = lower ? llvm::APIntOps::smax(maxOrMin, foldedResult)
2304 : llvm::APIntOps::smin(maxOrMin, foldedResult);
2306 lower ? forOp.setConstantLowerBound(maxOrMin.getSExtValue())
2307 : forOp.setConstantUpperBound(maxOrMin.getSExtValue());
2312 bool folded =
false;
2313 if (!forOp.hasConstantLowerBound())
2314 folded |=
succeeded(foldLowerOrUpperBound(
true));
2317 if (!forOp.hasConstantUpperBound())
2318 folded |=
succeeded(foldLowerOrUpperBound(
false));
2327 auto lbMap = forOp.getLowerBoundMap();
2328 auto ubMap = forOp.getUpperBoundMap();
2329 auto prevLbMap = lbMap;
2330 auto prevUbMap = ubMap;
2343 if (lbMap == prevLbMap && ubMap == prevUbMap)
2346 if (lbMap != prevLbMap)
2347 forOp.setLowerBound(lbOperands, lbMap);
2348 if (ubMap != prevUbMap)
2349 forOp.setUpperBound(ubOperands, ubMap);
2355 static std::optional<uint64_t> getTrivialConstantTripCount(AffineForOp forOp) {
2356 int64_t step = forOp.getStep();
2357 if (!forOp.hasConstantBounds() || step <= 0)
2358 return std::nullopt;
2359 int64_t lb = forOp.getConstantLowerBound();
2360 int64_t ub = forOp.getConstantUpperBound();
2361 return ub - lb <= 0 ? 0 : (ub - lb + step - 1) / step;
2372 if (!llvm::hasSingleElement(*forOp.getBody()))
2374 if (forOp.getNumResults() == 0)
2376 std::optional<uint64_t> tripCount = getTrivialConstantTripCount(forOp);
2377 if (tripCount && *tripCount == 0) {
2380 rewriter.
replaceOp(forOp, forOp.getIterOperands());
2384 auto yieldOp = cast<AffineYieldOp>(forOp.getBody()->getTerminator());
2385 auto iterArgs = forOp.getRegionIterArgs();
2386 bool hasValDefinedOutsideLoop =
false;
2387 bool iterArgsNotInOrder =
false;
2388 for (
unsigned i = 0, e = yieldOp->getNumOperands(); i < e; ++i) {
2389 Value val = yieldOp.getOperand(i);
2390 auto *iterArgIt = llvm::find(iterArgs, val);
2391 if (iterArgIt == iterArgs.end()) {
2393 assert(forOp.isDefinedOutsideOfLoop(val) &&
2394 "must be defined outside of the loop");
2395 hasValDefinedOutsideLoop =
true;
2396 replacements.push_back(val);
2398 unsigned pos = std::distance(iterArgs.begin(), iterArgIt);
2400 iterArgsNotInOrder =
true;
2401 replacements.push_back(forOp.getIterOperands()[pos]);
2406 if (!tripCount.has_value() &&
2407 (hasValDefinedOutsideLoop || iterArgsNotInOrder))
2411 if (tripCount.has_value() && tripCount.value() >= 2 && iterArgsNotInOrder)
2413 rewriter.
replaceOp(forOp, replacements);
2421 results.
add<AffineForEmptyLoopFolder>(context);
2429 AffineForOp::getSuccessorEntryOperands(std::optional<unsigned> index) {
2430 assert((!index || *index == 0) &&
"invalid region index");
2434 return getIterOperands();
2442 void AffineForOp::getSuccessorRegions(
2445 assert((!index.has_value() || index.value() == 0) &&
"expected loop region");
2450 std::optional<uint64_t> tripCount = getTrivialConstantTripCount(*
this);
2451 if (!index.has_value() && tripCount.has_value()) {
2452 if (tripCount.value() > 0) {
2453 regions.push_back(
RegionSuccessor(&getLoopBody(), getRegionIterArgs()));
2456 if (tripCount.value() == 0) {
2464 if (index && tripCount && *tripCount == 1) {
2471 regions.push_back(
RegionSuccessor(&getLoopBody(), getRegionIterArgs()));
2477 std::optional<uint64_t> tripCount = getTrivialConstantTripCount(op);
2478 return tripCount && *tripCount == 0;
2491 results.assign(getIterOperands().begin(), getIterOperands().end());
2498 auto lbMap = getLowerBoundMap();
2503 auto lbMap = getLowerBoundMap();
2504 auto ubMap = getUpperBoundMap();
2511 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
2516 newOperands.append(ubOperands.begin(), ubOperands.end());
2517 auto iterOperands = getIterOperands();
2518 newOperands.append(iterOperands.begin(), iterOperands.end());
2519 (*this)->setOperands(newOperands);
2526 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
2529 newOperands.append(ubOperands.begin(), ubOperands.end());
2530 auto iterOperands = getIterOperands();
2531 newOperands.append(iterOperands.begin(), iterOperands.end());
2532 (*this)->setOperands(newOperands);
2537 void AffineForOp::setLowerBoundMap(
AffineMap map) {
2538 auto lbMap = getLowerBoundMap();
2541 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
2546 void AffineForOp::setUpperBoundMap(
AffineMap map) {
2547 auto ubMap = getUpperBoundMap();
2550 assert(map.
getNumResults() >= 1 &&
"bound map has at least one result");
2555 bool AffineForOp::hasConstantLowerBound() {
2556 return getLowerBoundMap().isSingleConstant();
2559 bool AffineForOp::hasConstantUpperBound() {
2560 return getUpperBoundMap().isSingleConstant();
2563 int64_t AffineForOp::getConstantLowerBound() {
2564 return getLowerBoundMap().getSingleConstantResult();
2567 int64_t AffineForOp::getConstantUpperBound() {
2568 return getUpperBoundMap().getSingleConstantResult();
2571 void AffineForOp::setConstantLowerBound(int64_t value) {
2575 void AffineForOp::setConstantUpperBound(int64_t value) {
2580 return {operand_begin(), operand_begin() + getLowerBoundMap().getNumInputs()};
2584 return {operand_begin() + getLowerBoundMap().getNumInputs(),
2585 operand_begin() + getLowerBoundMap().getNumInputs() +
2586 getUpperBoundMap().getNumInputs()};
2589 AffineForOp::operand_range AffineForOp::getControlOperands() {
2590 return {operand_begin(), operand_begin() + getLowerBoundMap().getNumInputs() +
2591 getUpperBoundMap().getNumInputs()};
2594 bool AffineForOp::matchingBoundOperandList() {
2595 auto lbMap = getLowerBoundMap();
2596 auto ubMap = getUpperBoundMap();
2602 for (
unsigned i = 0, e = lbMap.
getNumInputs(); i < e; i++) {
2604 if (getOperand(i) != getOperand(numOperands + i))
2610 Region &AffineForOp::getLoopBody() {
return getRegion(); }
2612 std::optional<Value> AffineForOp::getSingleInductionVar() {
2613 return getInductionVar();
2616 std::optional<OpFoldResult> AffineForOp::getSingleLowerBound() {
2617 if (!hasConstantLowerBound())
2618 return std::nullopt;
2620 return OpFoldResult(b.getI64IntegerAttr(getConstantLowerBound()));
2623 std::optional<OpFoldResult> AffineForOp::getSingleStep() {
2628 std::optional<OpFoldResult> AffineForOp::getSingleUpperBound() {
2629 if (!hasConstantUpperBound())
2630 return std::nullopt;
2632 return OpFoldResult(b.getI64IntegerAttr(getConstantUpperBound()));
2660 auto ivArg = llvm::dyn_cast<BlockArgument>(val);
2661 if (!ivArg || !ivArg.getOwner())
2662 return AffineForOp();
2663 auto *containingInst = ivArg.getOwner()->getParent()->getParentOp();
2664 if (
auto forOp = dyn_cast<AffineForOp>(containingInst))
2666 return forOp.getInductionVar() == val ? forOp : AffineForOp();
2667 return AffineForOp();
2671 auto ivArg = llvm::dyn_cast<BlockArgument>(val);
2672 if (!ivArg || !ivArg.getOwner())
2675 auto parallelOp = dyn_cast<AffineParallelOp>(containingOp);
2676 if (parallelOp && llvm::is_contained(parallelOp.getIVs(), val))
2685 ivs->reserve(forInsts.size());
2686 for (
auto forInst : forInsts)
2687 ivs->push_back(forInst.getInductionVar());
2692 ivs.reserve(affineOps.size());
2695 if (
auto forOp = dyn_cast<AffineForOp>(op))
2696 ivs.push_back(forOp.getInductionVar());
2697 else if (
auto parallelOp = dyn_cast<AffineParallelOp>(op))
2698 for (
size_t i = 0; i < parallelOp.getBody()->getNumArguments(); i++)
2699 ivs.push_back(parallelOp.getBody()->getArgument(i));
2705 template <
typename BoundListTy,
typename LoopCreatorTy>
2710 LoopCreatorTy &&loopCreatorFn) {
2711 assert(lbs.size() == ubs.size() &&
"Mismatch in number of arguments");
2712 assert(lbs.size() == steps.size() &&
"Mismatch in number of arguments");
2724 ivs.reserve(lbs.size());
2725 for (
unsigned i = 0, e = lbs.size(); i < e; ++i) {
2731 if (i == e - 1 && bodyBuilderFn) {
2733 bodyBuilderFn(nestedBuilder, nestedLoc, ivs);
2735 nestedBuilder.
create<AffineYieldOp>(nestedLoc);
2740 auto loop = loopCreatorFn(builder, loc, lbs[i], ubs[i], steps[i], loopBody);
2748 int64_t ub, int64_t step,
2749 AffineForOp::BodyBuilderFn bodyBuilderFn) {
2750 return builder.
create<AffineForOp>(loc, lb, ub, step,
2751 std::nullopt, bodyBuilderFn);
2758 AffineForOp::BodyBuilderFn bodyBuilderFn) {
2761 if (lbConst && ubConst)
2763 ubConst.value(), step, bodyBuilderFn);
2766 std::nullopt, bodyBuilderFn);
2790 bool replaceLoopResults) {
2791 assert(newIterOperands.size() == newYieldedValues.size() &&
2792 "newIterOperands must be of the same size as newYieldedValues");
2796 auto operands = llvm::to_vector<4>(loop.getIterOperands());
2797 operands.append(newIterOperands.begin(), newIterOperands.end());
2801 auto lbMap = loop.getLowerBoundMap();
2802 auto ubMap = loop.getUpperBoundMap();
2803 AffineForOp newLoop =
2804 b.
create<AffineForOp>(loop.getLoc(), lbOperands, lbMap, ubOperands, ubMap,
2805 loop.getStep(), operands);
2807 newLoop.getLoopBody().takeBody(loop.getLoopBody());
2808 for (
Value val : newIterArgs)
2809 newLoop.getLoopBody().addArgument(val.
getType(), val.
getLoc());
2812 if (!newYieldedValues.empty()) {
2813 auto yield = cast<AffineYieldOp>(newLoop.getBody()->getTerminator());
2815 auto yieldOperands = llvm::to_vector<4>(yield.getOperands());
2816 yieldOperands.append(newYieldedValues.begin(), newYieldedValues.end());
2817 b.
create<AffineYieldOp>(yield.getLoc(), yieldOperands);
2820 if (replaceLoopResults) {
2821 for (
auto it : llvm::zip(loop.getResults(), newLoop.getResults().take_front(
2822 loop.getNumResults()))) {
2823 std::get<0>(it).replaceAllUsesWith(std::get<1>(it));
2840 if (ifOp.getElseRegion().empty() ||
2841 !llvm::hasSingleElement(*ifOp.getElseBlock()) || ifOp.getNumResults())
2859 auto isTriviallyFalse = [](
IntegerSet iSet) {
2860 return iSet.isEmptyIntegerSet();
2864 return (iSet.getNumEqualities() == 1 && iSet.getNumInequalities() == 0 &&
2865 iSet.getConstraint(0) == 0);
2868 IntegerSet affineIfConditions = op.getIntegerSet();
2870 if (isTriviallyFalse(affineIfConditions)) {
2880 blockToMove = op.getElseBlock();
2881 }
else if (isTriviallyTrue(affineIfConditions)) {
2882 blockToMove = op.getThenBlock();
2900 rewriter.
eraseOp(blockToMoveTerminator);
2908 void AffineIfOp::getSuccessorRegions(
2913 if (!index.has_value()) {
2918 if (!getElseRegion().empty())
2932 auto conditionAttr =
2933 (*this)->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName());
2935 return emitOpError(
"requires an integer set attribute named 'condition'");
2938 IntegerSet condition = conditionAttr.getValue();
2940 return emitOpError(
"operand count and condition integer set dimension and "
2941 "symbol count must match");
2953 IntegerSetAttr conditionAttr;
2956 AffineIfOp::getConditionAttrStrName(),
2962 auto set = conditionAttr.getValue();
2963 if (set.getNumDims() != numDims)
2966 "dim operand count and integer set dim count must match");
2967 if (numDims + set.getNumSymbols() != result.
operands.size())
2970 "symbol operand count and integer set symbol count must match");
2984 AffineIfOp::ensureTerminator(*thenRegion, parser.
getBuilder(),
2991 AffineIfOp::ensureTerminator(*elseRegion, parser.
getBuilder(),
3003 auto conditionAttr =
3004 (*this)->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName());
3005 p <<
" " << conditionAttr;
3007 conditionAttr.getValue().getNumDims(), p);
3014 auto &elseRegion = this->getElseRegion();
3015 if (!elseRegion.
empty()) {
3024 getConditionAttrStrName());
3029 ->getAttrOfType<IntegerSetAttr>(getConditionAttrStrName())
3033 void AffineIfOp::setIntegerSet(
IntegerSet newSet) {
3039 (*this)->setOperands(operands);
3044 bool withElseRegion) {
3045 assert(resultTypes.empty() || withElseRegion);
3052 if (resultTypes.empty())
3053 AffineIfOp::ensureTerminator(*thenRegion, builder, result.
location);
3056 if (withElseRegion) {
3058 if (resultTypes.empty())
3059 AffineIfOp::ensureTerminator(*elseRegion, builder, result.
location);
3065 AffineIfOp::build(builder, result, {}, set, args,
3080 if (llvm::none_of(operands,
3092 auto set = getIntegerSet();
3098 if (getIntegerSet() == set && llvm::equal(operands, getOperands()))
3101 setConditional(set, operands);
3107 results.
add<SimplifyDeadElse, AlwaysTrueOrFalseIf>(context);
3116 assert(operands.size() == 1 + map.
getNumInputs() &&
"inconsistent operands");
3120 auto memrefType = llvm::cast<MemRefType>(operands[0].getType());
3121 result.
types.push_back(memrefType.getElementType());
3126 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
3129 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
3131 result.
types.push_back(memrefType.getElementType());
3136 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
3137 int64_t rank = memrefType.getRank();
3142 build(builder, result, memref, map, indices);
3151 AffineMapAttr mapAttr;
3156 AffineLoadOp::getMapAttrStrName(),
3166 p <<
" " << getMemRef() <<
'[';
3167 if (AffineMapAttr mapAttr =
3168 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
3172 {getMapAttrStrName()});
3181 MemRefType memrefType,
unsigned numIndexOperands) {
3185 return op->
emitOpError(
"affine map num results must equal memref rank");
3187 return op->
emitOpError(
"expects as many subscripts as affine map inputs");
3189 if (memrefType.getRank() != numIndexOperands)
3191 "expects the number of subscripts to be equal to memref rank");
3195 for (
auto idx : mapOperands) {
3196 if (!idx.getType().isIndex())
3197 return op->
emitOpError(
"index to load must have 'index' type");
3199 return op->
emitOpError(
"index must be a dimension or symbol identifier");
3207 if (getType() != memrefType.getElementType())
3208 return emitOpError(
"result type must match element type of memref");
3212 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
3213 getMapOperands(), memrefType,
3214 getNumOperands() - 1)))
3222 results.
add<SimplifyAffineOp<AffineLoadOp>>(context);
3231 auto getGlobalOp = getMemref().getDefiningOp<memref::GetGlobalOp>();
3238 auto global = dyn_cast_or_null<memref::GlobalOp>(
3245 llvm::dyn_cast_or_null<DenseElementsAttr>(global.getConstantInitValue());
3249 if (
auto splatAttr = llvm::dyn_cast<SplatElementsAttr>(cstAttr))
3250 return splatAttr.getSplatValue<
Attribute>();
3252 if (!getAffineMap().isConstant())
3254 auto indices = llvm::to_vector<4>(
3255 llvm::map_range(getAffineMap().getConstantResults(),
3256 [](int64_t v) -> uint64_t {
return v; }));
3257 return cstAttr.getValues<
Attribute>()[indices];
3267 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
3278 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
3279 int64_t rank = memrefType.getRank();
3284 build(builder, result, valueToStore, memref, map, indices);
3293 AffineMapAttr mapAttr;
3298 mapOperands, mapAttr, AffineStoreOp::getMapAttrStrName(),
3309 p <<
" " << getValueToStore();
3310 p <<
", " << getMemRef() <<
'[';
3311 if (AffineMapAttr mapAttr =
3312 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
3316 {getMapAttrStrName()});
3323 if (getValueToStore().getType() != memrefType.getElementType())
3325 "value to store must have the same type as memref element type");
3329 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
3330 getMapOperands(), memrefType,
3331 getNumOperands() - 2)))
3339 results.
add<SimplifyAffineOp<AffineStoreOp>>(context);
3352 template <
typename T>
3356 op.getMap().getNumDims() + op.getMap().getNumSymbols())
3358 "operand count and affine map dimension and symbol count must match");
3362 template <
typename T>
3364 p <<
' ' << op->
getAttr(T::getMapAttrStrName());
3366 unsigned numDims = op.getMap().getNumDims();
3367 p <<
'(' << operands.take_front(numDims) <<
')';
3369 if (operands.size() != numDims)
3370 p <<
'[' << operands.drop_front(numDims) <<
']';
3372 {T::getMapAttrStrName()});
3375 template <
typename T>
3382 AffineMapAttr mapAttr;
3398 template <
typename T>
3400 static_assert(llvm::is_one_of<T, AffineMinOp, AffineMaxOp>::value,
3401 "expected affine min or max op");
3407 auto foldedMap = op.getMap().partialConstantFold(operands, &results);
3409 if (foldedMap.getNumSymbols() == 1 && foldedMap.isSymbolIdentity())
3413 if (results.empty()) {
3415 if (foldedMap == op.getMap())
3422 auto resultIt = std::is_same<T, AffineMinOp>::value
3423 ? std::min_element(results.begin(), results.end())
3424 : std::max_element(results.begin(), results.end());
3425 if (resultIt == results.end())
3431 template <
typename T>
3437 AffineMap oldMap = affineOp.getAffineMap();
3443 if (!llvm::is_contained(newExprs, expr))
3444 newExprs.push_back(expr);
3474 template <
typename T>
3480 AffineMap oldMap = affineOp.getAffineMap();
3482 affineOp.getMapOperands().take_front(oldMap.
getNumDims());
3484 affineOp.getMapOperands().take_back(oldMap.
getNumSymbols());
3486 auto newDimOperands = llvm::to_vector<8>(dimOperands);
3487 auto newSymOperands = llvm::to_vector<8>(symOperands);
3496 Value symValue = symOperands[symExpr.getPosition()];
3498 producerOps.push_back(producerOp);
3502 Value dimValue = dimOperands[dimExpr.getPosition()];
3504 producerOps.push_back(producerOp);
3511 newExprs.push_back(expr);
3514 if (producerOps.empty())
3521 for (T producerOp : producerOps) {
3522 AffineMap producerMap = producerOp.getAffineMap();
3523 unsigned numProducerDims = producerMap.
getNumDims();
3528 producerOp.getMapOperands().take_front(numProducerDims);
3530 producerOp.getMapOperands().take_back(numProducerSyms);
3531 newDimOperands.append(dimValues.begin(), dimValues.end());
3532 newSymOperands.append(symValues.begin(), symValues.end());
3536 newExprs.push_back(expr.shiftDims(numProducerDims, numUsedDims)
3537 .shiftSymbols(numProducerSyms, numUsedSyms));
3540 numUsedDims += numProducerDims;
3541 numUsedSyms += numProducerSyms;
3547 llvm::to_vector<8>(llvm::concat<Value>(newDimOperands, newSymOperands));
3566 if (!resultExpr.isPureAffine())
3582 if (llvm::is_sorted(flattenedExprs))
3587 llvm::to_vector(llvm::seq<unsigned>(0, map.
getNumResults()));
3588 llvm::sort(resultPermutation, [&](
unsigned lhs,
unsigned rhs) {
3589 return flattenedExprs[lhs] < flattenedExprs[rhs];
3592 for (
unsigned idx : resultPermutation)
3613 template <
typename T>
3619 AffineMap map = affineOp.getAffineMap();
3627 template <
typename T>
3633 if (affineOp.getMap().getNumResults() != 1)
3636 affineOp.getOperands());
3664 return parseAffineMinMaxOp<AffineMinOp>(parser, result);
3692 return parseAffineMinMaxOp<AffineMaxOp>(parser, result);
3711 IntegerAttr hintInfo;
3713 StringRef readOrWrite, cacheType;
3715 AffineMapAttr mapAttr;
3719 AffinePrefetchOp::getMapAttrStrName(),
3725 AffinePrefetchOp::getLocalityHintAttrStrName(),
3735 if (!readOrWrite.equals(
"read") && !readOrWrite.equals(
"write"))
3737 "rw specifier has to be 'read' or 'write'");
3739 AffinePrefetchOp::getIsWriteAttrStrName(),
3742 if (!cacheType.equals(
"data") && !cacheType.equals(
"instr"))
3744 "cache type has to be 'data' or 'instr'");
3747 AffinePrefetchOp::getIsDataCacheAttrStrName(),
3754 p <<
" " << getMemref() <<
'[';
3755 AffineMapAttr mapAttr =
3756 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName());
3759 p <<
']' <<
", " << (getIsWrite() ?
"write" :
"read") <<
", "
3760 <<
"locality<" << getLocalityHint() <<
">, "
3761 << (getIsDataCache() ?
"data" :
"instr");
3763 (*this)->getAttrs(),
3764 {getMapAttrStrName(), getLocalityHintAttrStrName(),
3765 getIsDataCacheAttrStrName(), getIsWriteAttrStrName()});
3770 auto mapAttr = (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName());
3774 return emitOpError(
"affine.prefetch affine map num results must equal"
3777 return emitOpError(
"too few operands");
3779 if (getNumOperands() != 1)
3780 return emitOpError(
"too few operands");
3784 for (
auto idx : getMapOperands()) {
3786 return emitOpError(
"index must be a dimension or symbol identifier");
3794 results.
add<SimplifyAffineOp<AffinePrefetchOp>>(context);
3812 auto ubs = llvm::to_vector<4>(llvm::map_range(ranges, [&](int64_t value) {
3816 build(builder, result, resultTypes, reductions, lbs, {}, ubs,
3826 assert(llvm::all_of(lbMaps,
3828 return m.
getNumDims() == lbMaps[0].getNumDims() &&
3831 "expected all lower bounds maps to have the same number of dimensions "
3833 assert(llvm::all_of(ubMaps,
3835 return m.
getNumDims() == ubMaps[0].getNumDims() &&
3838 "expected all upper bounds maps to have the same number of dimensions "
3840 assert((lbMaps.empty() || lbMaps[0].getNumInputs() == lbArgs.size()) &&
3841 "expected lower bound maps to have as many inputs as lower bound "
3843 assert((ubMaps.empty() || ubMaps[0].getNumInputs() == ubArgs.size()) &&
3844 "expected upper bound maps to have as many inputs as upper bound "
3851 for (arith::AtomicRMWKind reduction : reductions)
3852 reductionAttrs.push_back(
3864 groups.reserve(groups.size() + maps.size());
3865 exprs.reserve(maps.size());
3870 return AffineMap::get(maps[0].getNumDims(), maps[0].getNumSymbols(), exprs,
3871 maps[0].getContext());
3876 AffineMap lbMap = concatMapsSameInput(lbMaps, lbGroups);
3877 AffineMap ubMap = concatMapsSameInput(ubMaps, ubGroups);
3892 auto *body =
new Block();
3894 for (
unsigned i = 0, e = steps.size(); i < e; ++i)
3897 if (resultTypes.empty())
3898 ensureTerminator(*bodyRegion, builder, result.
location);
3901 Region &AffineParallelOp::getLoopBody() {
return getRegion(); }
3903 unsigned AffineParallelOp::getNumDims() {
return getSteps().size(); }
3905 AffineParallelOp::operand_range AffineParallelOp::getLowerBoundsOperands() {
3906 return getOperands().take_front(getLowerBoundsMap().getNumInputs());
3909 AffineParallelOp::operand_range AffineParallelOp::getUpperBoundsOperands() {
3910 return getOperands().drop_front(getLowerBoundsMap().getNumInputs());
3913 AffineMap AffineParallelOp::getLowerBoundMap(
unsigned pos) {
3914 auto values = getLowerBoundsGroups().getValues<int32_t>();
3916 for (
unsigned i = 0; i < pos; ++i)
3918 return getLowerBoundsMap().getSliceMap(start, values[pos]);
3921 AffineMap AffineParallelOp::getUpperBoundMap(
unsigned pos) {
3922 auto values = getUpperBoundsGroups().getValues<int32_t>();
3924 for (
unsigned i = 0; i < pos; ++i)
3926 return getUpperBoundsMap().getSliceMap(start, values[pos]);
3930 return AffineValueMap(getLowerBoundsMap(), getLowerBoundsOperands());
3934 return AffineValueMap(getUpperBoundsMap(), getUpperBoundsOperands());
3937 std::optional<SmallVector<int64_t, 8>> AffineParallelOp::getConstantRanges() {
3938 if (hasMinMaxBounds())
3939 return std::nullopt;
3944 AffineValueMap::difference(getUpperBoundsValueMap(), getLowerBoundsValueMap(),
3947 for (
unsigned i = 0, e = rangesValueMap.
getNumResults(); i < e; ++i) {
3948 auto expr = rangesValueMap.
getResult(i);
3951 return std::nullopt;
3952 out.push_back(cst.getValue());
3957 Block *AffineParallelOp::getBody() {
return &getRegion().
front(); }
3959 OpBuilder AffineParallelOp::getBodyBuilder() {
3960 return OpBuilder(getBody(), std::prev(getBody()->end()));
3965 "operands to map must match number of inputs");
3967 auto ubOperands = getUpperBoundsOperands();
3970 newOperands.append(ubOperands.begin(), ubOperands.end());
3971 (*this)->setOperands(newOperands);
3978 "operands to map must match number of inputs");
3981 newOperands.append(ubOperands.begin(), ubOperands.end());
3982 (*this)->setOperands(newOperands);
3988 setStepsAttr(getBodyBuilder().getI64ArrayAttr(newSteps));
3992 auto numDims = getNumDims();
3995 getSteps().size() != numDims || getBody()->getNumArguments() != numDims) {
3996 return emitOpError() <<
"the number of region arguments ("
3997 << getBody()->getNumArguments()
3998 <<
") and the number of map groups for lower ("
3999 << getLowerBoundsGroups().getNumElements()
4000 <<
") and upper bound ("
4001 << getUpperBoundsGroups().getNumElements()
4002 <<
"), and the number of steps (" << getSteps().size()
4003 <<
") must all match";
4006 unsigned expectedNumLBResults = 0;
4007 for (APInt v : getLowerBoundsGroups())
4008 expectedNumLBResults += v.getZExtValue();
4009 if (expectedNumLBResults != getLowerBoundsMap().getNumResults())
4010 return emitOpError() <<
"expected lower bounds map to have "
4011 << expectedNumLBResults <<
" results";
4012 unsigned expectedNumUBResults = 0;
4013 for (APInt v : getUpperBoundsGroups())
4014 expectedNumUBResults += v.getZExtValue();
4015 if (expectedNumUBResults != getUpperBoundsMap().getNumResults())
4016 return emitOpError() <<
"expected upper bounds map to have "
4017 << expectedNumUBResults <<
" results";
4019 if (getReductions().size() != getNumResults())
4020 return emitOpError(
"a reduction must be specified for each output");
4023 for (
Attribute attr : getReductions()) {
4024 auto intAttr = llvm::dyn_cast<IntegerAttr>(attr);
4025 if (!intAttr || !arith::symbolizeAtomicRMWKind(intAttr.getInt()))
4026 return emitOpError(
"invalid reduction attribute");
4032 getLowerBoundsMap().getNumDims())))
4036 getUpperBoundsMap().getNumDims())))
4043 auto newMap = getAffineMap();
4045 if (newMap == getAffineMap() && newOperands == operands)
4047 reset(newMap, newOperands);
4060 if (!lbCanonicalized && !ubCanonicalized)
4063 if (lbCanonicalized)
4065 if (ubCanonicalized)
4083 StringRef keyword) {
4086 ValueRange dimOperands = operands.take_front(numDims);
4087 ValueRange symOperands = operands.drop_front(numDims);
4089 for (llvm::APInt groupSize : group) {
4093 unsigned size = groupSize.getZExtValue();
4098 p << keyword <<
'(';
4108 p <<
" (" << getBody()->getArguments() <<
") = (";
4110 getLowerBoundsOperands(),
"max");
4113 getUpperBoundsOperands(),
"min");
4116 bool elideSteps = llvm::all_of(steps, [](int64_t step) {
return step == 1; });
4119 llvm::interleaveComma(steps, p);
4122 if (getNumResults()) {
4124 llvm::interleaveComma(getReductions(), p, [&](
auto &attr) {
4125 arith::AtomicRMWKind sym = *arith::symbolizeAtomicRMWKind(
4126 llvm::cast<IntegerAttr>(attr).getInt());
4127 p <<
"\"" << arith::stringifyAtomicRMWKind(sym) <<
"\"";
4129 p <<
") -> (" << getResultTypes() <<
")";
4136 (*this)->getAttrs(),
4137 {AffineParallelOp::getReductionsAttrStrName(),
4138 AffineParallelOp::getLowerBoundsMapAttrStrName(),
4139 AffineParallelOp::getLowerBoundsGroupsAttrStrName(),
4140 AffineParallelOp::getUpperBoundsMapAttrStrName(),
4141 AffineParallelOp::getUpperBoundsGroupsAttrStrName(),
4142 AffineParallelOp::getStepsAttrStrName()});
4155 "expected operands to be dim or symbol expression");
4158 for (
const auto &list : operands) {
4162 for (
Value operand : valueOperands) {
4163 unsigned pos = std::distance(uniqueOperands.begin(),
4164 llvm::find(uniqueOperands, operand));
4165 if (pos == uniqueOperands.size())
4166 uniqueOperands.push_back(operand);
4167 replacements.push_back(
4177 enum class MinMaxKind { Min, Max };
4201 const llvm::StringLiteral tmpAttrStrName =
"__pseudo_bound_map";
4203 StringRef mapName = kind == MinMaxKind::Min
4204 ? AffineParallelOp::getUpperBoundsMapAttrStrName()
4205 : AffineParallelOp::getLowerBoundsMapAttrStrName();
4206 StringRef groupsName =
4207 kind == MinMaxKind::Min
4208 ? AffineParallelOp::getUpperBoundsGroupsAttrStrName()
4209 : AffineParallelOp::getLowerBoundsGroupsAttrStrName();
4226 auto parseOperands = [&]() {
4228 kind == MinMaxKind::Min ?
"min" :
"max"))) {
4229 mapOperands.clear();
4236 llvm::append_range(flatExprs, map.getValue().getResults());
4238 auto dimsRef = operandsRef.take_front(map.getValue().getNumDims());
4241 auto symsRef = operandsRef.drop_front(map.getValue().getNumDims());
4244 flatDimOperands.append(map.getValue().getNumResults(), dims);
4245 flatSymOperands.append(map.getValue().getNumResults(), syms);
4246 numMapsPerGroup.push_back(map.getValue().getNumResults());
4249 flatSymOperands.emplace_back(),
4250 flatExprs.emplace_back())))
4252 numMapsPerGroup.push_back(1);
4259 unsigned totalNumDims = 0;
4260 unsigned totalNumSyms = 0;
4261 for (
unsigned i = 0, e = flatExprs.size(); i < e; ++i) {
4262 unsigned numDims = flatDimOperands[i].size();
4263 unsigned numSyms = flatSymOperands[i].size();
4264 flatExprs[i] = flatExprs[i]
4265 .shiftDims(numDims, totalNumDims)
4266 .shiftSymbols(numSyms, totalNumSyms);
4267 totalNumDims += numDims;
4268 totalNumSyms += numSyms;
4280 result.
operands.append(dimOperands.begin(), dimOperands.end());
4281 result.
operands.append(symOperands.begin(), symOperands.end());
4284 auto flatMap =
AffineMap::get(totalNumDims, totalNumSyms, flatExprs,
4286 flatMap = flatMap.replaceDimsAndSymbols(
4287 dimRplacements, symRepacements, dimOperands.size(), symOperands.size());
4311 AffineMapAttr stepsMapAttr;
4316 result.
addAttribute(AffineParallelOp::getStepsAttrStrName(),
4320 AffineParallelOp::getStepsAttrStrName(),
4327 auto stepsMap = stepsMapAttr.getValue();
4328 for (
const auto &result : stepsMap.getResults()) {
4332 "steps must be constant integers");
4333 steps.push_back(constExpr.getValue());
4335 result.
addAttribute(AffineParallelOp::getStepsAttrStrName(),
4355 std::optional<arith::AtomicRMWKind> reduction =
4356 arith::symbolizeAtomicRMWKind(attrVal.getValue());
4358 return parser.
emitError(loc,
"invalid reduction value: ") << attrVal;
4359 reductions.push_back(
4367 result.
addAttribute(AffineParallelOp::getReductionsAttrStrName(),
4376 for (
auto &iv : ivs)
4377 iv.type = indexType;
4383 AffineParallelOp::ensureTerminator(*body, builder, result.
location);
4392 auto *parentOp = (*this)->getParentOp();
4393 auto results = parentOp->getResults();
4394 auto operands = getOperands();
4396 if (!isa<AffineParallelOp, AffineIfOp, AffineForOp>(parentOp))
4397 return emitOpError() <<
"only terminates affine.if/for/parallel regions";
4398 if (parentOp->getNumResults() != getNumOperands())
4399 return emitOpError() <<
"parent of yield must have same number of "
4400 "results as the yield operands";
4401 for (
auto it : llvm::zip(results, operands)) {
4402 if (std::get<0>(it).getType() != std::get<1>(it).getType())
4403 return emitOpError() <<
"types mismatch between yield op and its parent";
4416 assert(operands.size() == 1 + map.
getNumInputs() &&
"inconsistent operands");
4420 result.
types.push_back(resultType);
4424 VectorType resultType,
Value memref,
4426 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
4430 result.
types.push_back(resultType);
4434 VectorType resultType,
Value memref,
4436 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
4437 int64_t rank = memrefType.getRank();
4442 build(builder, result, resultType, memref, map, indices);
4445 void AffineVectorLoadOp::getCanonicalizationPatterns(
RewritePatternSet &results,
4447 results.
add<SimplifyAffineOp<AffineVectorLoadOp>>(context);
4455 MemRefType memrefType;
4456 VectorType resultType;
4458 AffineMapAttr mapAttr;
4463 AffineVectorLoadOp::getMapAttrStrName(),
4474 p <<
" " << getMemRef() <<
'[';
4475 if (AffineMapAttr mapAttr =
4476 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
4480 {getMapAttrStrName()});
4486 VectorType vectorType) {
4488 if (memrefType.getElementType() != vectorType.getElementType())
4490 "requires memref and vector types of the same elemental type");
4498 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
4499 getMapOperands(), memrefType,
4500 getNumOperands() - 1)))
4516 assert(map.
getNumInputs() == mapOperands.size() &&
"inconsistent index info");
4527 auto memrefType = llvm::cast<MemRefType>(memref.
getType());
4528 int64_t rank = memrefType.getRank();
4533 build(builder, result, valueToStore, memref, map, indices);
4535 void AffineVectorStoreOp::getCanonicalizationPatterns(
4537 results.
add<SimplifyAffineOp<AffineVectorStoreOp>>(context);
4544 MemRefType memrefType;
4545 VectorType resultType;
4548 AffineMapAttr mapAttr;
4554 AffineVectorStoreOp::getMapAttrStrName(),
4565 p <<
" " << getValueToStore();
4566 p <<
", " << getMemRef() <<
'[';
4567 if (AffineMapAttr mapAttr =
4568 (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()))
4572 {getMapAttrStrName()});
4573 p <<
" : " <<
getMemRefType() <<
", " << getValueToStore().getType();
4579 *
this, (*this)->getAttrOfType<AffineMapAttr>(getMapAttrStrName()),
4580 getMapOperands(), memrefType,
4581 getNumOperands() - 2)))
4602 if (staticDim.has_value())
4605 return llvm::dyn_cast_if_present<Value>(ofr);
4611 if (getBasis().empty())
4612 return emitOpError(
"basis should not be empty");
4613 if (getNumResults() != getBasis().size())
4614 return emitOpError(
"should return an index for each basis element");
4622 #define GET_OP_CLASSES
4623 #include "mlir/Dialect/Affine/IR/AffineOps.cpp.inc"
static AffineForOp buildAffineLoopFromConstants(OpBuilder &builder, Location loc, int64_t lb, int64_t ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds known to be constants.
static void materializeConstants(OpBuilder &b, Location loc, ArrayRef< OpFoldResult > values, SmallVectorImpl< Operation * > &constants, SmallVectorImpl< Value > &actualValues)
Given a list of OpFoldResult, build the necessary operations to populate actualValues with values pro...
static bool hasTrivialZeroTripCount(AffineForOp op)
Returns true if the affine.for has zero iterations in trivial cases.
static void composeMultiResultAffineMap(AffineMap &map, SmallVectorImpl< Value > &operands)
Composes the given affine map with the given list of operands, pulling in the maps from any affine....
static void printAffineMinMaxOp(OpAsmPrinter &p, T op)
static bool remainsLegalAfterInline(Value value, Region *src, Region *dest, const IRMapping &mapping, function_ref< bool(Value, Region *)> legalityCheck)
Checks if value known to be a legal affine dimension or symbol in src region remains legal if the ope...
static void printMinMaxBound(OpAsmPrinter &p, AffineMapAttr mapAttr, DenseIntElementsAttr group, ValueRange operands, StringRef keyword)
Prints a lower(upper) bound of an affine parallel loop with max(min) conditions in it.
static void LLVM_ATTRIBUTE_UNUSED simplifyMapWithOperands(AffineMap &map, ArrayRef< Value > operands)
Simplify the map while exploiting information on the values in operands.
static std::optional< int64_t > getBoundForExpr(AffineExpr expr, unsigned numDims, unsigned numSymbols, ArrayRef< std::optional< int64_t >> constLowerBounds, ArrayRef< std::optional< int64_t >> constUpperBounds, bool isUpper)
Get a lower or upper (depending on isUpper) bound for expr while using the constant lower and upper b...
static OpFoldResult foldMinMaxOp(T op, ArrayRef< Attribute > operands)
Fold an affine min or max operation with the given operands.
static LogicalResult canonicalizeLoopBounds(AffineForOp forOp)
Canonicalize the bounds of the given loop.
static Value createFoldedComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ValueRange operandsRef)
Fully compose map with operands and canonicalize the result.
static void simplifyExprAndOperands(AffineExpr &expr, unsigned numDims, unsigned numSymbols, ArrayRef< Value > operands)
Simplify expr while exploiting information from the values in operands.
static bool isValidAffineIndexOperand(Value value, Region *region)
static void canonicalizeMapOrSetAndOperands(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
static void composeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Iterate over operands and fold away all those produced by an AffineApplyOp iteratively.
static std::optional< int64_t > getUpperBound(Value iv)
Gets the constant upper bound on an affine.for iv.
static ParseResult parseBound(bool isLower, OperationState &result, OpAsmParser &p)
Parse a for operation loop bounds.
static std::optional< int64_t > getLowerBound(Value iv)
Gets the constant lower bound on an iv.
static void composeSetAndOperands(IntegerSet &set, SmallVectorImpl< Value > &operands)
Compose any affine.apply ops feeding into operands of the integer set set by composing the maps of su...
static LogicalResult replaceDimOrSym(AffineMap *map, unsigned dimOrSymbolPosition, SmallVectorImpl< Value > &dims, SmallVectorImpl< Value > &syms)
Replace all occurrences of AffineExpr at position pos in map by the defining AffineApplyOp expression...
static void canonicalizePromotedSymbols(MapOrSet *mapOrSet, SmallVectorImpl< Value > *operands)
static LogicalResult verifyVectorMemoryOp(Operation *op, MemRefType memrefType, VectorType vectorType)
Verify common invariants of affine.vector_load and affine.vector_store.
static void simplifyMinOrMaxExprWithOperands(AffineMap &map, ArrayRef< Value > operands, bool isMax)
Simplify the expressions in map while making use of lower or upper bounds of its operands.
static ParseResult parseAffineMinMaxOp(OpAsmParser &parser, OperationState &result)
static bool isMemRefSizeValidSymbol(AnyMemRefDefOp memrefDefOp, unsigned index, Region *region)
Returns true if the 'index' dimension of the memref defined by memrefDefOp is a statically shaped one...
static bool isNonNegativeBoundedBy(AffineExpr e, ArrayRef< Value > operands, int64_t k)
Check if e is known to be: 0 <= e < k.
static ParseResult parseAffineMapWithMinMax(OpAsmParser &parser, OperationState &result, MinMaxKind kind)
Parses an affine map that can contain a min/max for groups of its results, e.g., max(expr-1,...
static AffineForOp buildAffineLoopFromValues(OpBuilder &builder, Location loc, Value lb, Value ub, int64_t step, AffineForOp::BodyBuilderFn bodyBuilderFn)
Creates an affine loop from the bounds that may or may not be constants.
static std::enable_if_t< OpTy::template hasTrait< OpTrait::OneResult >), OpFoldResult > createOrFold(OpBuilder &b, Location loc, ValueRange operands, Args &&...leadingArguments)
Create an operation of the type provided as template argument and attempt to fold it immediately.
static void printDimAndSymbolList(Operation::operand_iterator begin, Operation::operand_iterator end, unsigned numDims, OpAsmPrinter &printer)
Prints dimension and symbol list.
static int64_t getLargestKnownDivisor(AffineExpr e, ArrayRef< Value > operands)
Returns the largest known divisor of e.
static void buildAffineLoopNestImpl(OpBuilder &builder, Location loc, BoundListTy lbs, BoundListTy ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn, LoopCreatorTy &&loopCreatorFn)
Builds an affine loop nest, using "loopCreatorFn" to create individual loop operations.
static LogicalResult foldLoopBounds(AffineForOp forOp)
Fold the constant bounds of a loop.
static LogicalResult verifyDimAndSymbolIdentifiers(OpTy &op, Operation::operand_range operands, unsigned numDims)
Utility function to verify that a set of operands are valid dimension and symbol identifiers.
static OpFoldResult makeComposedFoldedMinMax(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
static bool isDimOpValidSymbol(ShapedDimOpInterface dimOp, Region *region)
Returns true if the result of the dim op is a valid symbol for region.
static bool isQTimesDPlusR(AffineExpr e, ArrayRef< Value > operands, int64_t &div, AffineExpr "ientTimesDiv, AffineExpr &rem)
Check if expression e is of the form d*e_1 + e_2 where 0 <= e_2 < d.
static ParseResult deduplicateAndResolveOperands(OpAsmParser &parser, ArrayRef< SmallVector< OpAsmParser::UnresolvedOperand >> operands, SmallVectorImpl< Value > &uniqueOperands, SmallVectorImpl< AffineExpr > &replacements, AffineExprKind kind)
Given a list of lists of parsed operands, populates uniqueOperands with unique operands.
static LogicalResult verifyAffineMinMaxOp(T op)
static void printBound(AffineMapAttr boundMap, Operation::operand_range boundOperands, const char *prefix, OpAsmPrinter &p)
static LogicalResult verifyMemoryOpIndexing(Operation *op, AffineMapAttr mapAttr, Operation::operand_range mapOperands, MemRefType memrefType, unsigned numIndexOperands)
Verify common indexing invariants of affine.load, affine.store, affine.vector_load and affine....
static LogicalResult canonicalizeMapExprAndTermOrder(AffineMap &map)
Canonicalize the result expression order of an affine map and return success if the order changed.
static Operation * materializeConstant(Dialect *dialect, OpBuilder &builder, Attribute value, Type type, Location loc)
A utility function used to materialize a constant for a given attribute and type.
static bool isLegalToInline(InlinerInterface &interface, Region *src, Region *insertRegion, bool shouldCloneInlinedRegion, IRMapping &valueMapping)
Utility to check that all of the operations within 'src' can be inlined.
static Operation::operand_range getLowerBoundOperands(AffineForOp forOp)
static Operation::operand_range getUpperBoundOperands(AffineForOp forOp)
static void print(spirv::VerCapExtAttr triple, DialectAsmPrinter &printer)
static VectorType getVectorType(Type scalarTy, const VectorizationStrategy *strategy)
Returns the vector type resulting from applying the provided vectorization strategy on the scalar typ...
static int64_t getNumElements(ShapedType type)
Affine binary operation expression.
An integer constant appearing in affine expression.
A dimensional identifier appearing in an affine expression.
RetTy walkPostOrder(AffineExpr expr)
Base type for affine expression.
AffineExpr floorDiv(uint64_t v) const
AffineExprKind getKind() const
Return the classification for this type.
int64_t getLargestKnownDivisor() const
Returns the greatest known integral divisor of this affine expression.
MLIRContext * getContext() const
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued.
AffineMap getSliceMap(unsigned start, unsigned length) const
Returns the map consisting of length expressions starting from start.
MLIRContext * getContext() const
bool isFunctionOfDim(unsigned position) const
Return true if any affine expression involves AffineDimExpr position.
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
AffineMap shiftDims(unsigned shift, unsigned offset=0) const
Replace dims[offset ...
unsigned getNumSymbols() const
unsigned getNumDims() const
ArrayRef< AffineExpr > getResults() const
bool isFunctionOfSymbol(unsigned position) const
Return true if any affine expression involves AffineSymbolExpr position.
LogicalResult constantFold(ArrayRef< Attribute > operandConstants, SmallVectorImpl< Attribute > &results) const
Folds the results of the application of an affine map on the provided operands to a constant if possi...
unsigned getNumResults() const
AffineMap replaceDimsAndSymbols(ArrayRef< AffineExpr > dimReplacements, ArrayRef< AffineExpr > symReplacements, unsigned numResultDims, unsigned numResultSyms) const
This method substitutes any uses of dimensions and symbols (e.g.
unsigned getNumInputs() const
AffineMap shiftSymbols(unsigned shift, unsigned offset=0) const
Replace symbols[offset ...
static SmallVector< AffineMap, 4 > inferFromExprList(ArrayRef< ArrayRef< AffineExpr >> exprsList)
Returns a vector of AffineMaps; each with as many results as exprs.size(), as many dims as the larges...
AffineExpr getResult(unsigned idx) const
AffineMap replace(AffineExpr expr, AffineExpr replacement, unsigned numResultDims, unsigned numResultSyms) const
Sparse replace method.
static AffineMap getConstantMap(int64_t val, MLIRContext *context)
Returns a single constant result affine map.
AffineMap getSubMap(ArrayRef< unsigned > resultPos) const
Returns the map consisting of the resultPos subset.
A symbolic identifier appearing in an affine expression.
@ Paren
Parens surrounding zero or more operands.
@ OptionalSquare
Square brackets supporting zero or more ops, or nothing.
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
virtual ParseResult parseRParen()=0
Parse a ) token.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
ParseResult addTypeToList(Type type, SmallVectorImpl< Type > &result)
Add the specified type to the end of the specified type list and return success.
virtual ParseResult parseOptionalRParen()=0
Parse a ) token if present.
virtual ParseResult parseLess()=0
Parse a '<' token.
virtual ParseResult parseEqual()=0
Parse a = token.
virtual ParseResult parseColonType(Type &result)=0
Parse a colon followed by a type.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseGreater()=0
Parse a '>' token.
virtual ParseResult parseLParen()=0
Parse a ( token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
virtual ParseResult parseComma()=0
Parse a , token.
virtual ParseResult parseOptionalArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an optional arrow followed by a type list.
virtual ParseResult parseArrowTypeList(SmallVectorImpl< Type > &result)=0
Parse an arrow followed by a type list.
ParseResult parseKeyword(StringRef keyword)
Parse a given keyword.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
void printOptionalArrowTypeList(TypeRange &&types)
Print an optional arrow followed by a type list.
Attributes are known-constant values of operations.
Block represents an ordered list of Operations.
Operation * getTerminator()
Get the terminator operation of this block.
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
BlockArgListType getArguments()
This class is a general helper class for creating context-global objects like types,...
IntegerAttr getIndexAttr(int64_t value)
IntegerAttr getIntegerAttr(Type type, int64_t value)
AffineMap getDimIdentityMap()
AffineMap getMultiDimIdentityMap(unsigned rank)
DenseIntElementsAttr getI32TensorAttr(ArrayRef< int32_t > values)
Tensor-typed DenseIntElementsAttr getters.
IntegerAttr getI64IntegerAttr(int64_t value)
IntegerType getIntegerType(unsigned width)
BoolAttr getBoolAttr(bool value)
AffineMap getEmptyAffineMap()
Returns a zero result affine map with no dimensions or symbols: () -> ().
AffineMap getConstantAffineMap(int64_t val)
Returns a single constant result affine map with 0 dimensions and 0 symbols.
MLIRContext * getContext() const
AffineMap getSymbolIdentityMap()
ArrayAttr getArrayAttr(ArrayRef< Attribute > value)
ArrayAttr getI64ArrayAttr(ArrayRef< int64_t > values)
An attribute that represents a reference to a dense integer vector or tensor object.
This is the interface that must be implemented by the dialects of operations to be inlined.
DialectInlinerInterface(Dialect *dialect)
This is a utility class for mapping one set of IR entities to another.
auto lookup(T from) const
Lookup a mapped value within the map.
An integer set representing a conjunction of one or more affine equalities and inequalities.
unsigned getNumDims() const
static IntegerSet get(unsigned dimCount, unsigned symbolCount, ArrayRef< AffineExpr > constraints, ArrayRef< bool > eqFlags)
MLIRContext * getContext() const
unsigned getNumInputs() const
ArrayRef< AffineExpr > getConstraints() const
ArrayRef< bool > getEqFlags() const
Returns the equality bits, which specify whether each of the constraints is an equality or inequality...
unsigned getNumSymbols() const
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
MLIRContext is the top-level object for a collection of MLIR operations.
Dialect * getLoadedDialect(StringRef name)
Get a registered IR dialect with the given namespace.
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
void pop_back()
Pop last element from list.
Attribute erase(StringAttr name)
Erase the attribute with the given name from the list.
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult parseRegion(Region ®ion, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region.
virtual ParseResult parseArgument(Argument &result, bool allowType=false, bool allowAttrs=false)=0
Parse a single argument with the following syntax:
ParseResult parseTrailingOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None)
Parse zero or more trailing SSA comma-separated trailing operand references with a specified surround...
virtual ParseResult parseArgumentList(SmallVectorImpl< Argument > &result, Delimiter delimiter=Delimiter::None, bool allowType=false, bool allowAttrs=false)=0
Parse zero or more arguments with a specified surrounding delimiter.
virtual ParseResult parseAffineMapOfSSAIds(SmallVectorImpl< UnresolvedOperand > &operands, Attribute &map, StringRef attrName, NamedAttrList &attrs, Delimiter delimiter=Delimiter::Square)=0
Parses an affine map attribute where dims and symbols are SSA operands.
ParseResult parseAssignmentList(SmallVectorImpl< Argument > &lhs, SmallVectorImpl< UnresolvedOperand > &rhs)
Parse a list of assignments of the form (x1 = y1, x2 = y2, ...)
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
ParseResult resolveOperands(Operands &&operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
virtual ParseResult parseAffineExprOfSSAIds(SmallVectorImpl< UnresolvedOperand > &dimOperands, SmallVectorImpl< UnresolvedOperand > &symbOperands, AffineExpr &expr)=0
Parses an affine expression where dims and symbols are SSA operands.
virtual ParseResult parseOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None, bool allowResultNumber=true, int requiredOperandCount=-1)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printAffineExprOfSSAIds(AffineExpr expr, ValueRange dimOperands, ValueRange symOperands)=0
Prints an affine expression of SSA ids with SSA id names used instead of dims and symbols.
virtual void printAffineMapOfSSAIds(AffineMapAttr mapAttr, ValueRange operands)=0
Prints an affine map of SSA ids, where SSA id names are used in place of dims/symbols.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
virtual void printRegionArgument(BlockArgument arg, ArrayRef< NamedAttribute > argAttrs={}, bool omitType=false)=0
Print a block argument in the usual format of: ssaName : type {attr1=42} loc("here") where location p...
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
void setListener(Listener *newListener)
Sets the listener of this builder to the one provided.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
Listener * getListener() const
Returns the current listener of this builder, or nullptr if this builder doesn't have a listener.
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
This class represents a single result from folding an operation.
A trait of region holding operations that defines a new scope for polyhedral optimization purposes.
This class provides the API for ops that are known to be isolated from above.
A trait used to provide symbol table functionalities to a region operation.
This class implements the operand iterators for the Operation class.
Operation is the basic unit of execution within MLIR.
LogicalResult fold(ArrayRef< Attribute > operands, SmallVectorImpl< OpFoldResult > &results)
Attempt to fold this operation with the specified constant operand values.
Value getOperand(unsigned idx)
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Attribute getAttr(StringAttr name)
Return the specified attribute if present, null otherwise.
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
MLIRContext * getContext()
Return the context this operation is associated with.
unsigned getNumOperands()
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
ArrayRef< NamedAttribute > getAttrs()
Return all of the attributes on this operation.
void setAttr(StringAttr name, Attribute value)
If the an attribute exists with the specified name, change it to the new value.
operand_range getOperands()
Returns an iterator on the underlying Value's.
Region * getParentRegion()
Returns the region to which the instruction belongs.
operand_range::iterator operand_iterator
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
void erase()
Remove this operation from its parent block and delete it.
unsigned getNumResults()
Return the number of results held by this operation.
This class represents success/failure for parsing-like operations that find it important to chain tog...
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
This class represents a successor of a region.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
void push_back(Block *block)
Operation * getParentOp()
Return the parent operation this region is attached to.
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
virtual void eraseBlock(Block *block)
This method erases all operations in a block.
virtual void replaceOp(Operation *op, ValueRange newValues)
This method replaces the results of the operation with the specified list of values.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
virtual void finalizeRootUpdate(Operation *op)
This method is used to signal the end of a root update on the given operation.
virtual void startRootUpdate(Operation *op)
This method is used to notify the rewriter that an in-place operation modification is about to happen...
virtual void inlineBlockBefore(Block *source, Block *dest, Block::iterator before, ValueRange argValues=std::nullopt)
Inline the operations of block 'source' into block 'dest' before the given position.
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replaces the result op with a new op that is created without verification.
This class represents a specific instance of an effect.
static DerivedEffect * get()
Returns a unique instance for the derived effect class.
static DefaultResource * get()
Returns a unique instance for the given effect class.
std::vector< SmallVector< int64_t, 8 > > operandExprStack
static Operation * lookupSymbolIn(Operation *op, StringAttr symbol)
Returns the operation registered with the given symbol name with the regions of 'symbolTableOp'.
This class provides an abstraction over the various different ranges of value types.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Location getLoc() const
Return the location of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
AffineBound represents a lower or upper bound in the for operation.
AffineDmaStartOp starts a non-blocking DMA operation that transfers data from a source memref to a de...
AffineDmaWaitOp blocks until the completion of a DMA operation associated with the tag element 'tag[i...
An AffineValueMap is an affine map plus its ML value operands and results for analysis purposes.
LogicalResult canonicalize()
Attempts to canonicalize the map and operands.
ArrayRef< Value > getOperands() const
AffineExpr getResult(unsigned i)
AffineMap getAffineMap() const
unsigned getNumResults() const
constexpr auto RecursivelySpeculatable
Speculatability
This enum is returned from the getSpeculatability method in the ConditionallySpeculatable op interfac...
constexpr auto NotSpeculatable
void buildAffineLoopNest(OpBuilder &builder, Location loc, ArrayRef< int64_t > lbs, ArrayRef< int64_t > ubs, ArrayRef< int64_t > steps, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuilderFn=nullptr)
Builds a perfect nest of affine.for loops, i.e., each loop except the innermost one contains only ano...
void fullyComposeAffineMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Given an affine map map and its input operands, this method composes into map, maps of AffineApplyOps...
void extractForInductionVars(ArrayRef< AffineForOp > forInsts, SmallVectorImpl< Value > *ivs)
Extracts the induction variables from a list of AffineForOps and places them in the output argument i...
bool isValidDim(Value value)
Returns true if the given Value can be used as a dimension id in the region of the closest surroundin...
SmallVector< OpFoldResult > makeComposedFoldedMultiResultAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Variant of makeComposedFoldedAffineApply suitable for multi-result maps.
bool isAffineInductionVar(Value val)
Returns true if the provided value is the induction variable of an AffineForOp or AffineParallelOp.
AffineApplyOp makeComposedAffineApply(OpBuilder &b, Location loc, AffineMap map, ValueRange operands)
Returns a composed AffineApplyOp by composing map and operands with other AffineApplyOps supplying th...
AffineForOp getForInductionVarOwner(Value val)
Returns the loop parent of an induction variable.
void canonicalizeMapAndOperands(AffineMap *map, SmallVectorImpl< Value > *operands)
Modifies both map and operands in-place so as to:
Value makeComposedAffineMin(OpBuilder &b, Location loc, AffineMap map, ValueRange operands)
Returns an AffineMinOp obtained by composing map and operands with AffineApplyOps supplying those ope...
OpFoldResult makeComposedFoldedAffineMax(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineMinOp that computes a maximum across the results of applying map to operands,...
bool isAffineForInductionVar(Value val)
Returns true if the provided value is the induction variable of an AffineForOp.
OpFoldResult makeComposedFoldedAffineMin(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineMinOp that computes a minimum across the results of applying map to operands,...
bool isTopLevelValue(Value value)
A utility function to check if a value is defined at the top level of an op with trait AffineScope or...
void canonicalizeSetAndOperands(IntegerSet *set, SmallVectorImpl< Value > *operands)
Canonicalizes an integer set the same way canonicalizeMapAndOperands does for affine maps.
void extractInductionVars(ArrayRef< Operation * > affineOps, SmallVectorImpl< Value > &ivs)
Extracts the induction variables from a list of either AffineForOp or AffineParallelOp and places the...
bool isValidSymbol(Value value)
Returns true if the given value can be used as a symbol in the region of the closest surrounding op t...
AffineForOp replaceForOpWithNewYields(OpBuilder &b, AffineForOp loop, ValueRange newIterOperands, ValueRange newYieldedValues, ValueRange newIterArgs, bool replaceLoopResults=true)
Replace loop with a new loop where newIterOperands are appended with new initialization values and ne...
OpFoldResult makeComposedFoldedAffineApply(OpBuilder &b, Location loc, AffineMap map, ArrayRef< OpFoldResult > operands)
Constructs an AffineApplyOp that applies map to operands after composing the map with the maps of any...
AffineParallelOp getAffineParallelInductionVarOwner(Value val)
Returns true if the provided value is among the induction variables of an AffineParallelOp.
SmallVector< Value, 4 > applyMapToValues(OpBuilder &b, Location loc, AffineMap map, ValueRange values)
Returns the values obtained by applying map to the list of values.
Region * getAffineScope(Operation *op)
Returns the closest region enclosing op that is held by an operation with trait AffineScope; nullptr ...
ParseResult parseDimAndSymbolList(OpAsmParser &parser, SmallVectorImpl< Value > &operands, unsigned &numDims)
Parses dimension and symbol list.
bool isAffineParallelInductionVar(Value val)
Returns true if val is the induction variable of an AffineParallelOp.
BaseMemRefType getMemRefType(Value value, const BufferizationOptions &options, MemRefLayoutAttrInterface layout={}, Attribute memorySpace=nullptr)
Return a MemRefType to which the type of the given value can be bufferized.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
LogicalResult foldMemRefCast(Operation *op, Value inner=nullptr)
This is a common utility used for patterns of the form "someop(memref.cast) -> someop".
LLVM_ATTRIBUTE_ALWAYS_INLINE MPInt gcd(const MPInt &a, const MPInt &b)
This header declares functions that assit transformations in the MemRef dialect.
AffineMap simplifyAffineMap(AffineMap map)
Simplifies an affine map by simplifying its underlying AffineExpr results.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
AffineMap removeDuplicateExprs(AffineMap map)
Returns a map with the same dimension and symbol count as map, but whose results are the unique affin...
std::optional< int64_t > getConstantIntValue(OpFoldResult ofr)
If ofr is a constant integer or an IntegerAttr, return the integer.
int64_t floorDiv(int64_t lhs, int64_t rhs)
Returns the result of MLIR's floordiv operation on constants.
int64_t ceilDiv(int64_t lhs, int64_t rhs)
Returns the result of MLIR's ceildiv operation on constants.
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
@ CeilDiv
RHS of ceildiv is always a constant or a symbolic expression.
@ Mod
RHS of mod is always a constant or a symbolic expression with a positive value.
@ DimId
Dimensional identifier.
@ FloorDiv
RHS of floordiv is always a constant or a symbolic expression.
@ SymbolId
Symbolic identifier.
AffineExpr getAffineBinaryOpExpr(AffineExprKind kind, AffineExpr lhs, AffineExpr rhs)
AffineExpr getAffineConstantExpr(int64_t constant, MLIRContext *context)
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
bool isStrided(MemRefType t)
Return true if the layout for t is compatible with strided semantics.
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
AffineExpr getAffineDimExpr(unsigned position, MLIRContext *context)
These free functions allow clients of the API to not use classes in detail.
LogicalResult verify(Operation *op, bool verifyRecursively=true)
Perform (potentially expensive) checks of invariants, used to detect compiler bugs,...
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
AffineExpr getAffineSymbolExpr(unsigned position, MLIRContext *context)
int64_t mod(int64_t lhs, int64_t rhs)
Returns MLIR's mod operation on constants.
Canonicalize the affine map result expression order of an affine min/max operation.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Remove duplicated expressions in affine min/max ops.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
Merge an affine min/max op to its consumers if its consumer is also an affine min/max op.
LogicalResult matchAndRewrite(T affineOp, PatternRewriter &rewriter) const override
This class represents an efficient way to signal success or failure.
This is the representation of an operand reference.
This class represents a listener that may be used to hook into various actions within an OpBuilder.
virtual void notifyOperationInserted(Operation *op)
Notification handler for when an operation is inserted into the builder.
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
This represents an operation in an abstracted form, suitable for use with the builder APIs.
SmallVector< Value, 4 > operands
void addOperands(ValueRange newOperands)
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
void addTypes(ArrayRef< Type > newTypes)
SmallVector< std::unique_ptr< Region >, 1 > regions
Regions that the op will hold.
SmallVector< Type, 4 > types
Types of the results of this operation.
Region * addRegion()
Create a region that should be attached to the operation.