32 #include "llvm/ADT/SmallBitVector.h" 52 enum Reduction { kNoReduc, kSum, kProduct, kAnd, kOr, kXor };
58 :
options(o), loops(numLoops), sizes(numLoops), buffers(numTensors),
59 pointers(numTensors, std::vector<Value>(numLoops)),
60 indices(numTensors, std::vector<Value>(numLoops)),
61 highs(numTensors, std::vector<Value>(numLoops)),
62 pidxs(numTensors, std::vector<Value>(numLoops)),
63 idxs(numTensors, std::vector<Value>(numLoops)), redVal(), sparseOut(op),
64 outerParNest(nest), lexIdx(), lexVal(), expValues(), expFilled(),
65 expAdded(), expCount(), curVecMask() {}
71 std::vector<Value> loops;
72 std::vector<Value> sizes;
75 std::vector<Value> buffers;
78 std::vector<std::vector<Value>> pointers;
79 std::vector<std::vector<Value>> indices;
82 std::vector<std::vector<Value>> highs;
83 std::vector<std::vector<Value>> pidxs;
84 std::vector<std::vector<Value>> idxs;
87 unsigned redExp = -1u;
95 unsigned outerParNest;
103 unsigned curVecLength = 1;
116 std::vector<unsigned> &topSort) {
117 unsigned sz = topSort.size();
118 assert(m.
getNumResults() == sz &&
"TopoSort/AffineMap size mismatch");
122 for (
unsigned i = 0; i < sz; i++)
126 for (
unsigned i = 0; i < sz; i++)
127 perm[i] = inv[topSort[i]];
132 static unsigned perm(
const SparseTensorEncodingAttr &enc,
unsigned d) {
134 auto order = enc.getDimOrdering();
136 assert(order.isPermutation());
137 return order.getDimPosition(d);
144 static Dim toDim(
const SparseTensorEncodingAttr &enc,
unsigned d) {
147 if (tp == SparseTensorEncodingAttr::DimLevelType::Compressed)
149 if (tp == SparseTensorEncodingAttr::DimLevelType::Singleton)
166 merger.
setDim(tensor, idx, dim);
174 return findAffine(merger, tensor, binOp.getLHS(), dim, isDense) &&
175 findAffine(merger, tensor, binOp.getRHS(), dim, isDense);
190 bool annotated =
false;
191 for (
OpOperand *t : op.getInputAndOutputOperands()) {
192 auto map = op.getTiedIndexingMap(t);
196 assert(map.getNumResults() == op.getRank(t));
197 for (
unsigned d = 0, rank = map.getNumResults(); d < rank; d++) {
198 unsigned tensor = t->getOperandNumber();
211 std::vector<unsigned> &topSort,
212 std::vector<std::vector<bool>> &adjM) {
214 return visit[i] != 1;
216 for (
unsigned j = 0, e = visit.size();
j < e;
j++)
221 topSort.push_back(i);
236 adjM[fidx][idx] =
true;
257 std::vector<unsigned> &topSort,
unsigned mask,
261 unsigned n = op.getNumLoops();
262 std::vector<std::vector<bool>> adjM(n, std::vector<bool>(n,
false));
265 for (
OpOperand *t : op.getInputAndOutputOperands()) {
270 auto map = op.getTiedIndexingMap(t);
272 assert(map.getNumDims() == n);
274 if (!(mask & SortMask::kIncludeDense) && !enc)
280 for (
unsigned d = 1, rank = map.getNumResults(); d < rank; d++) {
287 if (mask & SortMask::kIncludeUndef) {
288 unsigned tensor = t->getOperandNumber();
289 for (
unsigned i = 0; i < n; i++)
291 for (
unsigned j = 0;
j < n;
j++)
301 std::vector<unsigned>
visit(n, 0);
302 for (
unsigned i = 0; i < n; i++)
306 std::reverse(std::begin(topSort), std::end(topSort));
313 if (
auto funcOp = dyn_cast<func::FuncOp>(arg.getOwner()->getParentOp()))
314 if (
auto attr = funcOp.getArgAttrOfType<
BoolAttr>(
316 bufferization::BufferizableOpInterface::kInplaceableAttrName))
317 return attr.getValue();
333 std::vector<unsigned> &topSort,
unsigned exp,
335 unsigned &outerParNest) {
345 bool allDense =
true;
346 auto iteratorTypes = op.iterator_types().getValue();
347 unsigned numLoops = iteratorTypes.size();
348 for (
unsigned i = 0; i < numLoops; i++)
365 for (
unsigned i = 0; i < numLoops; i++) {
373 if (nest >= op.getRank(lhs) - 1) {
392 return vector::CombiningKind::ADD;
394 return vector::CombiningKind::MUL;
396 return vector::CombiningKind::AND;
398 return vector::CombiningKind::OR;
400 return vector::CombiningKind::XOR;
402 llvm_unreachable(
"unknown reduction kind");
426 llvm_unreachable(
"unexpected reduction operator");
436 Value r = codegen.redVal;
437 switch (codegen.redKind) {
443 return builder.
create<vector::InsertElementOp>(
448 return builder.
create<vector::InsertElementOp>(
453 return builder.
create<vector::BroadcastOp>(loc, vtp, r);
455 llvm_unreachable(
"unknown reduction kind");
462 return builder.
create<vector::ReductionOp>(loc, kind, codegen.redVal);
467 assert(codegen.redKind != kNoReduc);
468 codegen.redVal = merger.
exp(codegen.redExp).
val = reduc;
482 linalg::GenericOp op, MemRefType denseTp,
487 bool isInit = op.isInitTensor(lhs);
497 builder.
create<bufferization::ToMemrefOp>(loc, denseTp, tensor);
509 Value alloc = builder.
create<memref::AllocOp>(loc, denseTp, args);
515 builder.
create<bufferization::ToMemrefOp>(loc, denseTp, tensor);
516 builder.
create<memref::CopyOp>(loc, init, alloc);
525 linalg::GenericOp op) {
527 assert(op.getNumInputsAndOutputs() == op.getNumInputs() + 1);
531 for (
OpOperand *t : op.getInputAndOutputOperands()) {
532 unsigned tensor = t->getOperandNumber();
533 auto shape = op.getShape(t);
534 auto map = op.getTiedIndexingMap(t);
538 for (
unsigned d = 0, rank = map.getNumResults(); d < rank; d++) {
545 auto dynShape = {ShapedType::kDynamicSize};
552 codegen.pointers[tensor][idx] =
553 builder.
create<ToPointersOp>(loc, ptrTp, t->get(), dim);
554 codegen.indices[tensor][idx] =
555 builder.
create<ToIndicesOp>(loc, indTp, t->get(), dim);
558 unsigned p =
perm(enc, d);
560 if (ShapedType::isDynamic(shape[p]))
562 assert(codegen.highs[tensor][idx] ==
nullptr);
563 codegen.sizes[idx] = codegen.highs[tensor][idx] = up;
572 auto denseTp = MemRefType::get(shape, elementType);
573 if (tensor < op.getNumInputs())
574 codegen.buffers[tensor] =
575 builder.
create<bufferization::ToMemrefOp>(loc, denseTp, t->get());
577 codegen.buffers[tensor] =
579 }
else if (t == codegen.sparseOut) {
582 auto dynShape = {ShapedType::kDynamicSize};
583 auto memTp = MemRefType::get(dynShape, builder.
getIndexType());
584 codegen.lexIdx = builder.
create<memref::AllocaOp>(loc, memTp, rank);
585 codegen.lexVal = builder.
create<memref::AllocaOp>(
586 loc, MemRefType::get({}, elementType));
589 auto dynShape = {ShapedType::kDynamicSize};
590 auto sparseTp = MemRefType::get(dynShape, elementType);
591 codegen.buffers[tensor] =
592 builder.
create<ToValuesOp>(loc, sparseTp, t->get());
599 unsigned numScalableDims = codegen.options.enableVLAVectorization;
600 return VectorType::get(codegen.curVecLength, etp, numScalableDims);
617 IntegerAttr loInt, hiInt, stepInt;
621 if (((hiInt.getInt() - loInt.getInt()) % stepInt.getInt()) == 0)
622 return builder.
create<vector::BroadcastOp>(
636 return builder.
create<vector::CreateMaskOp>(loc, mtp, end);
645 if (args.back().getType().isa<VectorType>()) {
647 Value indexVec = args.back();
649 return builder.
create<vector::GatherOp>(loc, vtp, ptr, scalarArgs, indexVec,
650 codegen.curVecMask, pass);
652 return builder.
create<vector::MaskedLoadOp>(loc, vtp, ptr, args,
653 codegen.curVecMask, pass);
660 if (args.back().getType().isa<VectorType>()) {
662 Value indexVec = args.back();
664 builder.
create<vector::ScatterOp>(loc, ptr, scalarArgs, indexVec,
665 codegen.curVecMask, rhs);
668 builder.
create<vector::MaskedStoreOp>(loc, ptr, args, codegen.curVecMask,
677 return builder.
create<vector::BroadcastOp>(val.
getLoc(), vtp, val);
689 return codegen.loops[idx];
693 return builder.
create<arith::AddIOp>(
694 loc,
genAffine(codegen, builder, binOp.getLHS(), loc),
695 genAffine(codegen, builder, binOp.getRHS(), loc));
699 return builder.
create<arith::MulIOp>(
700 loc,
genAffine(codegen, builder, binOp.getLHS(), loc),
701 genAffine(codegen, builder, binOp.getRHS(), loc));
708 llvm_unreachable(
"unexpected affine subscript");
714 auto map = op.getTiedIndexingMap(t);
719 return codegen.loops[idx];
727 auto map = op.getTiedIndexingMap(t);
729 unsigned rank = map.getNumResults();
736 assert(codegen.pidxs[tensor][idx] !=
nullptr);
737 args.push_back(codegen.pidxs[tensor][idx]);
739 for (
unsigned d = 0; d < rank; d++) {
741 args.push_back(
genAffine(codegen, builder, a, op.getLoc()));
744 return codegen.buffers[tensor];
752 if (!codegen.expValues) {
758 return builder.
create<memref::LoadOp>(loc, codegen.expValues, index);
766 if (!codegen.expValues) {
767 builder.
create<memref::StoreOp>(loc, rhs, codegen.lexVal);
768 builder.
create<LexInsertOp>(loc, t->
get(), codegen.lexIdx, codegen.lexVal);
781 Value filled = builder.
create<memref::LoadOp>(loc, codegen.expFilled, index);
782 Value cond = builder.
create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
788 builder.
create<memref::StoreOp>(loc, tval, codegen.expFilled, index);
789 builder.
create<memref::StoreOp>(loc, index, codegen.expAdded,
792 Value add = builder.
create<arith::AddIOp>(loc, codegen.expCount, one);
793 builder.
create<scf::YieldOp>(loc, add);
796 builder.
create<scf::YieldOp>(loc, codegen.expCount);
799 codegen.expCount = ifOp.getResult(0);
800 builder.
create<memref::StoreOp>(loc, rhs, codegen.expValues, index);
805 linalg::GenericOp op,
unsigned exp) {
809 if (codegen.curVecLength > 1 && !val.
getType().
isa<VectorType>())
815 if (t == codegen.sparseOut)
820 if (codegen.curVecLength > 1)
822 return builder.
create<memref::LoadOp>(op.getLoc(), ptr, args);
827 linalg::GenericOp op,
unsigned exp,
Value rhs) {
830 if (codegen.redVal) {
831 if (codegen.curVecLength > 1)
832 rhs = builder.
create<arith::SelectOp>(loc, codegen.curVecMask, rhs,
839 if (t == codegen.sparseOut) {
852 if (codegen.curVecLength > 1)
855 builder.
create<memref::StoreOp>(loc, rhs, ptr, args);
865 if (codegen.curVecLength > 1) {
879 if (!etp.
isa<IndexType>()) {
881 vload = builder.
create<arith::ExtUIOp>(
884 !codegen.options.enableSIMDIndex32)
885 vload = builder.
create<arith::ExtUIOp>(
894 Value load = builder.
create<memref::LoadOp>(loc, ptr, s);
908 if (codegen.curVecLength > 1)
916 Value mul = builder.
create<arith::MulIOp>(loc, size, p);
919 builder.
create<arith::IndexCastOp>(loc, vtp.getElementType(), mul);
922 return builder.
create<arith::AddIOp>(loc, mul, i);
928 Value ival = codegen.loops[idx];
933 unsigned vl = codegen.curVecLength;
934 if (vl > 1 && !itype.
isa<VectorType>()) {
937 ival = builder.
create<vector::BroadcastOp>(loc, vtp, ival);
940 if (vtp.isScalable()) {
942 Value stepv = builder.
create<LLVM::StepVectorOp>(loc, stepvty);
943 incr = builder.
create<arith::IndexCastOp>(loc, vtp, stepv);
946 for (
unsigned i = 0; i < vl; i++)
947 integers.push_back(APInt(64, i));
949 incr = builder.
create<arith::ConstantOp>(loc, vtp, values);
951 ival = builder.
create<arith::AddIOp>(loc, ival, incr);
965 if (
auto indexOp = dyn_cast<linalg::IndexOp>(def))
967 if (def->getBlock() == block) {
968 for (
unsigned i = 0, n = def->getNumOperands(); i < n; i++)
970 i,
relinkBranch(codegen, rewriter, block, def->getOperand(i), ldx));
978 linalg::GenericOp op,
unsigned exp,
unsigned ldx) {
986 if (merger.
exp(exp).
kind == Kind::kIndex)
996 ee =
relinkBranch(codegen, rewriter, ee.getParentBlock(), ee, ldx);
1002 unsigned ldx,
bool &atLevel) {
1008 return codegen.loops[idx] !=
nullptr;
1023 linalg::GenericOp op,
unsigned exp,
unsigned ldx,
1029 bool atLevel = ldx == -1u;
1031 auto map = op.getTiedIndexingMap(t);
1033 for (
unsigned d = 0, rank = map.getNumResults(); d < rank; d++) {
1041 OpOperand *lhs = op.getOutputOperand(0);
1047 codegen.redExp = exp;
1050 Value redVal = codegen.redVal;
1052 codegen.redExp = -1u;
1053 codegen.redKind = kNoReduc;
1062 merger.
exp(exp).
kind != Kind::kIndex) {
1069 genInvariants(merger, codegen, builder, op, e0, ldx, atStart, last);
1070 genInvariants(merger, codegen, builder, op, e1, ldx, atStart, last);
1076 linalg::GenericOp op,
unsigned at,
bool atStart) {
1078 if (!lhs || codegen.outerParNest != op.getRank(lhs) - 1 ||
1079 at != codegen.outerParNest)
1085 auto dynShape = {ShapedType::kDynamicSize};
1087 Type t1 = MemRefType::get(dynShape, etp);
1093 assert(res.getNumResults() == 4);
1094 assert(!codegen.expValues);
1095 codegen.expValues = res.getResult(0);
1096 codegen.expFilled = res.getResult(1);
1097 codegen.expAdded = res.getResult(2);
1098 codegen.expCount = res.getResult(3);
1100 assert(codegen.expValues);
1101 builder.
create<CompressOp>(loc, tensor, codegen.lexIdx, codegen.expValues,
1102 codegen.expFilled, codegen.expAdded,
1104 codegen.expValues = codegen.expFilled = codegen.expAdded =
1105 codegen.expCount =
Value();
1113 linalg::GenericOp op, std::vector<unsigned> &topSort,
1114 unsigned at, BitVector &inits) {
1115 bool needsUniv =
false;
1117 unsigned idx = topSort[at];
1120 for (
unsigned b = 0, be = inits.size(); b < be; b++) {
1122 unsigned tensor = merger.
tensor(b);
1123 assert(idx == merger.
index(b));
1127 for (; pat != 0; pat--) {
1128 if (codegen.pidxs[tensor][topSort[pat - 1]])
1131 Value ptr = codegen.pointers[tensor][idx];
1134 : codegen.pidxs[tensor][topSort[pat - 1]];
1135 codegen.pidxs[tensor][idx] =
genLoad(codegen, builder, loc, ptr, p0);
1136 Value p1 = builder.
create<arith::AddIOp>(loc, p0, one);
1137 codegen.highs[tensor][idx] =
genLoad(codegen, builder, loc, ptr, p1);
1153 static bool isVectorFor(CodeGen &codegen,
bool isInner,
bool isReduction,
1156 if (codegen.sparseOut && !isReduction)
1159 switch (codegen.options.vectorizationStrategy) {
1163 return isInner && !isSparse;
1167 llvm_unreachable(
"unexpected vectorization strategy");
1174 bool isSparse,
bool isVector) {
1176 if (codegen.sparseOut)
1179 switch (codegen.options.parallelizationStrategy) {
1183 return isOuter && !isSparse && !isReduction && !isVector;
1185 return isOuter && !isReduction && !isVector;
1187 return !isSparse && !isReduction && !isVector;
1189 return !isReduction && !isVector;
1191 llvm_unreachable(
"unexpected parallelization strategy");
1200 for (
OpOperand *t : op.getInputAndOutputOperands()) {
1202 auto map = op.getTiedIndexingMap(t);
1203 for (
unsigned d = 0, rank = map.getNumResults(); d < rank; d++) {
1221 linalg::GenericOp op,
bool isOuter,
bool isInner,
1222 unsigned idx, BitVector &indices) {
1223 unsigned fb = indices.find_first();
1224 unsigned tensor = merger.
tensor(fb);
1225 assert(idx == merger.
index(fb));
1226 auto iteratorTypes = op.iterator_types().getValue();
1229 bool isVector =
isVectorFor(codegen, isInner, isReduction, isSparse) &&
1232 isParallelFor(codegen, isOuter, isReduction, isSparse, isVector);
1236 codegen.curVecLength = codegen.options.vectorLength;
1240 Value lo = isSparse ? codegen.pidxs[tensor][idx] : codegen.loops[idx];
1241 Value hi = isSparse ? codegen.highs[tensor][idx] : codegen.sizes[idx];
1243 if (isVector && codegen.options.enableVLAVectorization) {
1244 Value vscale = builder.
create<vector::VectorScaleOp>(
1246 step = builder.
create<arith::MulIOp>(loc, vscale, step);
1252 scf::ParallelOp parOp = builder.
create<scf::ParallelOp>(loc, lo, hi, step);
1254 codegen.pidxs[tensor][idx] = parOp.getInductionVars()[0];
1256 codegen.loops[idx] = parOp.getInductionVars()[0];
1263 if (codegen.redVal) {
1265 if (isVector && !codegen.redVal.getType().isa<VectorType>()) {
1266 VectorType vtp =
vectorType(codegen, codegen.redVal.getType());
1270 operands.push_back(codegen.redVal);
1272 if (codegen.expValues)
1273 operands.push_back(codegen.expCount);
1274 scf::ForOp forOp = builder.
create<scf::ForOp>(loc, lo, hi, step, operands);
1276 updateReduc(merger, codegen, forOp.getRegionIterArgs().front());
1277 if (codegen.expValues)
1278 codegen.expCount = forOp.getRegionIterArgs().back();
1280 Value iv = forOp.getInductionVar();
1282 codegen.pidxs[tensor][idx] = iv;
1284 codegen.loops[idx] = iv;
1288 codegen.curVecMask =
genVectorMask(codegen, builder, iv, lo, hi, step);
1294 linalg::GenericOp op,
unsigned idx,
bool needsUniv,
1295 BitVector &indices) {
1300 for (
unsigned b = 0, be = indices.size(); b < be; b++) {
1302 unsigned tensor = merger.
tensor(b);
1303 assert(idx == merger.
index(b));
1304 types.push_back(indexType);
1305 operands.push_back(codegen.pidxs[tensor][idx]);
1308 if (codegen.redVal) {
1309 types.push_back(codegen.redVal.getType());
1310 operands.push_back(codegen.redVal);
1312 if (codegen.expValues) {
1313 types.push_back(indexType);
1314 operands.push_back(codegen.expCount);
1317 types.push_back(indexType);
1318 operands.push_back(codegen.loops[idx]);
1320 assert(types.size() == operands.size());
1322 scf::WhileOp whileOp = builder.
create<scf::WhileOp>(loc, types, operands);
1325 Block *before = builder.
createBlock(&whileOp.getBefore(), {}, types, locs);
1333 for (
unsigned b = 0, be = indices.size(); b < be; b++) {
1335 unsigned tensor = merger.
tensor(b);
1336 assert(idx == merger.
index(b));
1337 Value op1 = before->getArgument(o);
1338 Value op2 = codegen.highs[tensor][idx];
1339 Value opc = builder.
create<arith::CmpIOp>(loc, arith::CmpIPredicate::ult,
1341 cond = cond ? builder.
create<arith::AndIOp>(loc, cond, opc) : opc;
1342 codegen.pidxs[tensor][idx] = after->
getArgument(o++);
1347 if (codegen.expValues)
1351 assert(o == operands.size());
1352 builder.
create<scf::ConditionOp>(loc, cond, before->getArguments());
1360 linalg::GenericOp op, std::vector<unsigned> &topSort,
1361 unsigned at,
bool needsUniv, BitVector &indices) {
1362 unsigned idx = topSort[at];
1363 if (indices.count() == 1) {
1364 bool isOuter = at == 0;
1365 bool isInner = at == topSort.size() - 1;
1366 return genFor(merger, codegen, builder, op, isOuter, isInner, idx, indices);
1368 return genWhile(merger, codegen, builder, op, idx, needsUniv, indices);
1374 linalg::GenericOp op, std::vector<unsigned> &topSort,
1375 unsigned at,
bool needsUniv, BitVector &locals) {
1377 unsigned idx = topSort[at];
1381 for (
unsigned b = 0, be = locals.size(); b < be; b++) {
1383 unsigned tensor = merger.
tensor(b);
1384 assert(idx == merger.
index(b));
1385 Value ptr = codegen.indices[tensor][idx];
1386 Value s = codegen.pidxs[tensor][idx];
1388 codegen.idxs[tensor][idx] = load;
1392 loc, arith::CmpIPredicate::ult, load,
min);
1393 min = builder.
create<arith::SelectOp>(loc, cmp, load,
min);
1404 codegen.loops[idx] =
min;
1410 for (
unsigned b = 0, be = locals.size(); b < be; b++) {
1412 merger.
isDim(b, Dim::kDense)) {
1413 unsigned tensor = merger.
tensor(b);
1414 assert(idx == merger.
index(b));
1416 for (; pat != 0; pat--)
1417 if (codegen.pidxs[tensor][topSort[pat - 1]])
1420 : codegen.pidxs[tensor][topSort[pat - 1]];
1422 codegen, builder, loc, codegen.sizes[idx], p, codegen.loops[idx]);
1428 if (codegen.sparseOut && !codegen.expValues) {
1430 builder.
create<memref::StoreOp>(loc, codegen.loops[idx], codegen.lexIdx,
1437 OpBuilder &builder, linalg::GenericOp op,
1438 unsigned idx,
bool needsUniv,
1439 BitVector &induction, scf::WhileOp whileOp) {
1442 if (codegen.redVal || codegen.expValues) {
1443 while (
auto ifOp = dyn_cast_or_null<scf::IfOp>(
1447 if (codegen.redVal) {
1448 yields.push_back(codegen.redVal);
1449 updateReduc(merger, codegen, ifOp.getResult(y++));
1451 if (codegen.expValues) {
1452 yields.push_back(codegen.expCount);
1453 codegen.expCount = ifOp->getResult(y++);
1455 assert(y == yields.size());
1456 builder.
create<scf::YieldOp>(loc, yields);
1469 for (
unsigned b = 0, be = induction.size(); b < be; b++) {
1471 unsigned tensor = merger.
tensor(b);
1472 assert(idx == merger.
index(b));
1473 Value op1 = codegen.idxs[tensor][idx];
1474 Value op2 = codegen.loops[idx];
1475 Value op3 = codegen.pidxs[tensor][idx];
1476 Value cmp = builder.
create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
1478 Value add = builder.
create<arith::AddIOp>(loc, op3, one);
1479 operands.push_back(builder.
create<arith::SelectOp>(loc, cmp, add, op3));
1480 codegen.pidxs[tensor][idx] = whileOp->getResult(o++);
1483 if (codegen.redVal) {
1484 operands.push_back(codegen.redVal);
1485 updateReduc(merger, codegen, whileOp->getResult(o++));
1487 if (codegen.expValues) {
1488 operands.push_back(codegen.expCount);
1489 codegen.expCount = whileOp->getResult(o++);
1493 builder.
create<arith::AddIOp>(loc, codegen.loops[idx], one));
1494 codegen.loops[idx] = whileOp->getResult(o++);
1496 assert(o == operands.size());
1497 builder.
create<scf::YieldOp>(loc, operands);
1503 OpBuilder &builder, linalg::GenericOp op,
1508 if (codegen.redVal) {
1509 operands.push_back(codegen.redVal);
1512 if (codegen.expValues) {
1513 operands.push_back(codegen.expCount);
1514 codegen.expCount = loop->
getResult(o++);
1516 assert(o == operands.size());
1518 builder.
create<scf::YieldOp>(loc, operands);
1524 linalg::GenericOp op,
unsigned idx,
1525 BitVector &conditions) {
1529 for (
unsigned b = 0, be = conditions.size(); b < be; b++) {
1530 if (conditions[b]) {
1531 unsigned tensor = merger.
tensor(b);
1532 assert(idx == merger.
index(b));
1535 Value op1 = codegen.idxs[tensor][idx];
1536 Value op2 = codegen.loops[idx];
1537 clause = builder.
create<arith::CmpIOp>(loc, arith::CmpIPredicate::eq,
1542 cond = cond ? builder.
create<arith::AndIOp>(loc, cond, clause) : clause;
1546 types.push_back(codegen.redVal.getType());
1547 if (codegen.expValues)
1549 scf::IfOp ifOp = builder.
create<scf::IfOp>(loc, types, cond,
true);
1556 linalg::GenericOp op, scf::IfOp ifOp,
Operation *loop,
1559 if (codegen.redVal) {
1560 operands.push_back(codegen.redVal);
1563 if (codegen.expValues) {
1564 operands.push_back(codegen.expCount);
1565 codegen.expCount = cntInput;
1567 if (!operands.empty())
1568 builder.
create<scf::YieldOp>(op.getLoc(), operands);
1579 linalg::GenericOp op, std::vector<unsigned> &topSort,
1580 unsigned exp,
unsigned at,
unsigned idx,
unsigned ldx,
1582 assert(codegen.curVecLength == 1);
1583 assert(!codegen.loops[idx]);
1585 genInvariants(merger, codegen, builder, op, exp, ldx,
true);
1589 unsigned l0 = merger.
set(lts)[0];
1591 genInit(merger, codegen, builder, op, topSort, at, merger.
lat(l0).
bits);
1595 unsigned lsize = merger.
set(lts).size();
1596 for (
unsigned i = 1; i < lsize; i++) {
1597 unsigned li = merger.
set(lts)[i];
1607 OpBuilder &builder, linalg::GenericOp op,
1608 std::vector<unsigned> &topSort,
unsigned at,
1609 unsigned li,
bool needsUniv) {
1610 assert(codegen.curVecLength == 1);
1615 genLocals(merger, codegen, builder, op, topSort, at, needsUniv,
1622 linalg::GenericOp op,
Operation *loop,
unsigned idx,
1623 unsigned li,
bool needsUniv) {
1624 codegen.curVecLength = 1;
1626 if (
auto whileOp = dyn_cast<scf::WhileOp>(loop)) {
1628 merger.
lat(li).
bits, whileOp);
1638 linalg::GenericOp op,
unsigned exp,
unsigned at,
1639 unsigned idx,
unsigned ldx) {
1640 assert(codegen.curVecLength == 1);
1641 codegen.loops[idx] =
Value();
1644 if (
auto vtp = codegen.redVal.getType().dyn_cast<VectorType>())
1648 genInvariants(merger, codegen, builder, op, exp, ldx,
false);
1657 linalg::GenericOp op, std::vector<unsigned> &topSort,
1658 unsigned exp,
unsigned at) {
1660 if (at == topSort.size()) {
1661 unsigned ldx = topSort[at - 1];
1662 Value rhs =
genExp(merger, codegen, rewriter, op, exp, ldx);
1668 unsigned idx = topSort[at];
1669 unsigned ldx = at == 0 ? -1u : topSort[at - 1];
1673 bool needsUniv =
startLoopSeq(merger, codegen, rewriter, op, topSort, exp, at,
1677 unsigned lsize = merger.
set(lts).size();
1678 for (
unsigned i = 0; i < lsize; i++) {
1680 unsigned li = merger.
set(lts)[i];
1682 startLoop(merger, codegen, rewriter, op, topSort, at, li, needsUniv);
1686 Value redInput = codegen.redVal;
1687 Value cntInput = codegen.expCount;
1688 bool isWhile = dyn_cast<scf::WhileOp>(loop) !=
nullptr;
1689 for (
unsigned j = 0;
j < lsize;
j++) {
1690 unsigned lj = merger.
set(lts)[
j];
1691 unsigned ej = merger.
lat(lj).
exp;
1692 if (li == lj || merger.
latGT(li, lj)) {
1696 genIf(merger, codegen, rewriter, op, idx, merger.
lat(lj).
simple);
1697 genStmt(merger, codegen, rewriter, op, topSort, ej, at + 1);
1698 endIf(merger, codegen, rewriter, op, ifOp, loop, redInput, cntInput);
1700 genStmt(merger, codegen, rewriter, op, topSort, ej, at + 1);
1707 endLoop(merger, codegen, rewriter, op, loop, idx, li, needsUniv);
1711 endLoopSeq(merger, codegen, rewriter, op, exp, at, idx, ldx);
1716 linalg::GenericOp op) {
1717 OpOperand *lhs = op.getOutputOperand(0);
1723 codegen.sparseOut == lhs);
1727 Value val = codegen.buffers.back();
1748 assert(op.getNumOutputs() == 1);
1749 unsigned numTensors = op.getNumInputsAndOutputs();
1750 unsigned numLoops = op.iterator_types().getValue().size();
1751 Merger merger(numTensors, numLoops);
1760 std::vector<unsigned> topSort;
1765 return resolveCycle(merger, rewriter, op);
1770 if (!optExp.hasValue())
1772 unsigned exp = optExp.getValue();
1776 unsigned outerParNest = 0;
1783 CodeGen codegen(
options, numTensors, numLoops, sparseOut, outerParNest);
1785 genStmt(merger, codegen, rewriter, op, topSort, exp, 0);
1786 genResult(merger, codegen, rewriter, op);
1793 linalg::GenericOp op)
const {
1797 std::vector<unsigned> topSort;
1798 for (
OpOperand *t : op.getInputOperands()) {
1799 unsigned tensor = t->getOperandNumber();
1800 Value tval = t->get();
1812 auto srcTp = tval.
getType().
cast<RankedTensorType>();
1813 auto dstEnc = SparseTensorEncodingAttr::get(
1814 op->getContext(), srcEnc.getDimLevelType(),
1815 permute(getContext(), op.getTiedIndexingMap(t), topSort),
1816 srcEnc.getPointerBitWidth(), srcEnc.getIndexBitWidth());
1817 auto dstTp = RankedTensorType::get(srcTp.getShape(),
1818 srcTp.getElementType(), dstEnc);
1819 auto convert = rewriter.
create<ConvertOp>(tval.
getLoc(), dstTp, tval);
static void genInsertionStore(CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, OpOperand *t, Value rhs)
Generates insertion code to implement dynamic tensor store.
Affine binary operation expression.
Kind
Tensor expression kind.
TODO: Remove this file when SCCP and integer range analysis have been ported to the new framework...
static void genLocals(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, std::vector< unsigned > &topSort, unsigned at, bool needsUniv, BitVector &locals)
Generates the local variables for this loop, consisting of the sparse indices, restored universal den...
static bool topSortDFS(unsigned i, std::vector< unsigned > &visit, std::vector< unsigned > &topSort, std::vector< std::vector< bool >> &adjM)
A DFS helper to compute a topological sort.
static void updateReduc(Merger &merger, CodeGen &codegen, Value reduc)
Updates scalarized reduction value.
static bool genInit(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, std::vector< unsigned > &topSort, unsigned at, BitVector &inits)
Generates initialization code for the subsequent loop sequence at current index level.
MLIRContext * getContext() const
bool isOutTensor(unsigned b, unsigned i) const
Returns true if bit corresponds to index of output tensor.
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
static bool isMaterializing(Value val)
Returns true if tensor materializes uninitialized into the computation.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
static Value genExp(Merger &merger, CodeGen &codegen, RewriterBase &rewriter, linalg::GenericOp op, unsigned exp, unsigned ldx)
Recursively generates tensor expression.
Block * getInsertionBlock() const
Return the block the current insertion point belongs to.
Operation is a basic unit of execution within MLIR.
unsigned tensor(unsigned b) const
Bit translation.
bool isDim(unsigned b, Dim d) const
Returns true if bit corresponds to queried dim.
static Value genVectorInvariantValue(CodeGen &codegen, OpBuilder &builder, Value val)
Generates a vectorized invariant.
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
static void endLoopSeq(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, unsigned exp, unsigned at, unsigned idx, unsigned ldx)
Ends a loop sequence at given level.
bool isFunctionOfDim(unsigned position) const
Return true if the affine expression involves AffineDimExpr position.
static scf::IfOp genIf(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, unsigned idx, BitVector &conditions)
Generates a single if-statement within a while-loop.
TensorExp & exp(unsigned e)
Convenience getters to immediately access the stored nodes.
static bool computeIterationGraph(Merger &merger, linalg::GenericOp op, std::vector< unsigned > &topSort, unsigned mask, OpOperand *skip=nullptr)
Computes a topologically sorted iteration graph for the linalg operation.
Block represents an ordered list of Operations.
AffineExpr getAffineSymbolExpr(unsigned position)
static AffineMap getPermutationMap(ArrayRef< unsigned > permutation, MLIRContext *context)
Returns an AffineMap representing a permutation.
static DenseElementsAttr get(ShapedType type, ArrayRef< Attribute > values)
Constructs a dense elements attribute from an array of element values.
Value constantIndex(OpBuilder &builder, Location loc, int64_t i)
Generates a constant of index type.
static Type getElementType(Type type, ArrayRef< int32_t > indices, function_ref< InFlightDiagnostic(StringRef)> emitErrorFn)
Walks the given type hierarchy with the given indices, potentially down to component granularity...
static bool isParallelFor(CodeGen &codegen, bool isOuter, bool isReduction, bool isSparse, bool isVector)
Returns parallelization strategy.
static unsigned perm(const SparseTensorEncodingAttr &enc, unsigned d)
Helper method to apply dimension ordering permutation.
Value constantZero(OpBuilder &builder, Location loc, Type tp)
Generates a 0-valued constant of the given type.
BitVector bits
Conjunction of tensor loop indices as bitvector.
bool latGT(unsigned i, unsigned j) const
Returns true if Li > Lj.
DimLevelType
This enum mimics SparseTensorEncodingAttr::DimLevelType for breaking dependency cycles.
BlockArgument getArgument(unsigned i)
An integer constant appearing in affine expression.
static Operation * genWhile(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, unsigned idx, bool needsUniv, BitVector &indices)
Emit a while-loop for co-iteration over multiple indices.
unsigned optimizeSet(unsigned s0)
Optimizes the iteration lattice points in the given set.
static Operation * genFor(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, bool isOuter, bool isInner, unsigned idx, BitVector &indices)
Generates a for-loop on a single index.
unsigned exp
Index of the tensor expression.
SmallVector< unsigned, 16 > & set(unsigned s)
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
static Operation * startLoop(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, std::vector< unsigned > &topSort, unsigned at, unsigned li, bool needsUniv)
Starts a single loop in current sequence.
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
static Reduction getReduction(Kind kind)
Maps operation to reduction.
Value constantI1(OpBuilder &builder, Location loc, bool b)
Generates a constant of i1 type.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
unsigned getOperandNumber()
Return which operand this is in the OpOperand list of the Operation.
This class represents an efficient way to signal success or failure.
static void genVectorStore(CodeGen &codegen, OpBuilder &builder, Value rhs, Value ptr, ArrayRef< Value > args)
Generates a vectorized store a[ind[lo:hi]] = rhs or a[lo:hi] = rhs.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
static AffineMap permute(MLIRContext *context, AffineMap m, std::vector< unsigned > &topSort)
Helper method to construct a permuted dimension ordering that adheres to the given topological sort...
static bool denseUnitStrides(Merger &merger, linalg::GenericOp op, unsigned idx)
Checks unit stride for dense tensors.
static void genInvariants(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, unsigned exp, unsigned ldx, bool atStart, Kind last=Kind::kTensor)
Hoists loop invariant tensor loads for which indices have been exhausted.
static Value genInvariantValue(Merger &merger, CodeGen &codegen, OpBuilder &builder, unsigned exp)
Generates an invariant value.
Type getElementTypeOrSelf(Type type)
Return the element type or return the type itself.
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
static bool isAdmissableTensorExp(Merger &merger, linalg::GenericOp op, std::vector< unsigned > &topSort, unsigned exp, OpOperand **sparseOut, unsigned &outerParNest)
Returns true when the tensor expression is admissable for codegen.
static void endIf(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, scf::IfOp ifOp, Operation *loop, Value redInput, Value cntInput)
Generates end of true branch of if-statement within a while-loop.
static void genExpansion(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, unsigned at, bool atStart)
Generates an expanded access pattern in innermost dimension.
Special case of IntegerAttr to represent boolean integers, i.e., signless i1 integers.
unsigned tensor
Expressions representing tensors simply have a tensor number.
static Operation * genLoop(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, std::vector< unsigned > &topSort, unsigned at, bool needsUniv, BitVector &indices)
Generates a for-loop or a while-loop, depending on whether it implements singleton iteration or co-it...
Base type for affine expression.
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
This class provides an abstraction over the various different ranges of value types.
RHS of mul is always a constant or a symbolic expression.
Type getPointerOverheadType(Builder &builder, const SparseTensorEncodingAttr &enc)
Returns the mlir::Type for pointer overhead storage.
static void genBuffers(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op)
Local bufferization of all dense and sparse data structures.
Kind kind
Tensor expression kind.
static void genForInduction(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, Operation *loop)
Generates the induction structure for a for-loop.
unsigned getNumResults() const
static bool findAffine(Merger &merger, unsigned tensor, AffineExpr a, Dim dim, bool isDense)
Helper method to inspect affine expressions.
static Value genVectorReducEnd(CodeGen &codegen, OpBuilder &builder, Location loc, VectorType vtp)
Generates final value for a vector reduction.
IRValueT get() const
Return the current value being used by this operand.
static Value genIndexValue(CodeGen &codegen, OpBuilder &builder, unsigned idx, unsigned ldx)
Generates an index value.
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued...
static bool findSparseAnnotations(Merger &merger, linalg::GenericOp op)
Helper method to inspect sparse encodings in the tensor types.
LatPoint & lat(unsigned l)
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
This class represents an argument of a Block.
static Value genIndex(CodeGen &codegen, linalg::GenericOp op, OpOperand *t)
Generates index for load/store on sparse tensor.
static Value genLoad(CodeGen &codegen, OpBuilder &builder, Location loc, Value ptr, Value s)
Generates a pointer/index load from the sparse storage scheme.
Eliminates variable at the specified position using Fourier-Motzkin variable elimination.
void setOperand(unsigned idx, Value value)
Location getLoc() const
Return the location of this value.
Value constantOne(OpBuilder &builder, Location loc, Type tp)
Generates a 1-valued constant of the given type.
static Value genAffine(CodeGen &codegen, OpBuilder &builder, AffineExpr a, Location loc)
Generates an affine expression.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
SparseTensorEncodingAttr getSparseTensorEncoding(Type type)
Convenience method to get a sparse encoding attribute from a type.
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
bool isReductionIterator(Attribute attr)
void setHasSparseOut(bool s)
unsigned index(unsigned b) const
BitVector simple
Simplified conjunction of tensor loop indices as bitvector.
unsigned getDimPosition(unsigned idx) const
Extracts the position of the dimensional expression at the given result, when the caller knows it is ...
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
static void genStmt(Merger &merger, CodeGen &codegen, RewriterBase &rewriter, linalg::GenericOp op, std::vector< unsigned > &topSort, unsigned exp, unsigned at)
Recursively generates code while computing iteration lattices in order to manage the complexity of im...
static llvm::ManagedStatic< PassManagerOptions > options
static bool isInPlace(Value val)
Returns true if tensor has an in-place annotation.
static Value genInsertionLoad(CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, OpOperand *t)
Generates insertion code to implement dynamic tensor load.
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
AffineExprKind getKind() const
Return the classification for this type.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
Dim
Dimension level type for a tensor (undef means index does not appear).
Type getType() const
Return the type of this value.
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&... args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments...
static Value relinkBranch(CodeGen &codegen, RewriterBase &rewriter, Block *block, Value e, unsigned ldx)
Semi-ring branches are simply inlined by the sparse compiler.
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replaces the result op with a new op that is created without verification.
static bool isVectorFor(CodeGen &codegen, bool isInner, bool isReduction, bool isSparse)
Returns vectorization strategy.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
A dimensional identifier appearing in an affine expression.
static VectorType vectorType(CodeGen &codegen, Type etp)
Constructs vector type.
static Dim toDim(const SparseTensorEncodingAttr &enc, unsigned d)
Helper method to translate dim level type to internal representation.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
void populateSparsificationPatterns(RewritePatternSet &patterns, const SparsificationOptions &options=SparsificationOptions())
Sets up sparsification rewriting rules with the given options.
static Value genSubscript(CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, OpOperand *t, SmallVector< Value, 4 > &args)
Generates subscript for load/store on a dense or sparse tensor.
MLIRContext is the top-level object for a collection of MLIR operations.
static Value genVectorMask(CodeGen &codegen, OpBuilder &builder, Value iv, Value lo, Value hi, Value step)
Constructs vector iteration mask.
This class represents an operand of an operation.
unsigned index
Indices hold the index number.
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
AffineExpr getAffineDimExpr(unsigned position)
static Value genVectorReducInit(CodeGen &codegen, OpBuilder &builder, Location loc, VectorType vtp)
Generates an initial value for a vector reduction, following the scheme given in Chapter 5 of "The So...
static void genTensorStore(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, unsigned exp, Value rhs)
Generates a store on a dense or sparse tensor.
static Value genOutputBuffer(CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, MemRefType denseTp, ArrayRef< Value > args)
Generates buffer for the output tensor.
static vector::CombiningKind getCombiningKind(Reduction kind)
Maps reduction kind to vector::CombiningKind.
void setInsertionPointToEnd(Block *block)
Sets the insertion point to the end of the specified block.
static Value genAddress(CodeGen &codegen, OpBuilder &builder, Location loc, Value size, Value p, Value i)
Generates an address computation "sz * p + i".
static void addAffineOrderings(std::vector< std::vector< bool >> &adjM, AffineExpr a, AffineExpr b, unsigned fidx)
Helper method to add all constraints from the indices in one affine expression before all indices in ...
static Value genTensorLoad(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, unsigned exp)
Generates a load on a dense or sparse tensor.
static void genResult(Merger &merger, CodeGen &codegen, RewriterBase &rewriter, linalg::GenericOp op)
Converts the result computed by the sparse kernel into the required form.
unsigned buildLattices(unsigned e, unsigned i)
Builds the iteration lattices in a bottom-up traversal given the remaining tensor (sub)expression and...
bool isSingleCondition(unsigned t, unsigned e) const
Returns true if given tensor iterates only in the given tensor expression.
Type getIndexOverheadType(Builder &builder, const SparseTensorEncodingAttr &enc)
Returns the mlir::Type for index overhead storage.
Value val
Direct link to IR for an invariant or the destination value (to infer destination type) of a cast ope...
Options for the Sparsification pass.
Block * createBlock(Region *parent, Region::iterator insertPt={}, TypeRange argTypes=llvm::None, ArrayRef< Location > locs=llvm::None)
Add new block with 'argTypes' arguments and set the insertion point to the end of it...
static Value genVectorLoad(CodeGen &codegen, OpBuilder &builder, Value ptr, ArrayRef< Value > args)
Generates a vectorized load lhs = a[ind[lo:hi]] or lhs = a[lo:hi].
static bool startLoopSeq(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, std::vector< unsigned > &topSort, unsigned exp, unsigned at, unsigned idx, unsigned ldx, unsigned lts)
Starts a loop sequence at given level.
Value createOrFoldDimOp(OpBuilder &b, Location loc, Value source, int64_t dim)
Helper function that creates a memref::DimOp or tensor::DimOp depending on the type of source...
static void genWhileInduction(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, unsigned idx, bool needsUniv, BitVector &induction, scf::WhileOp whileOp)
Generates the induction structure for a while-loop.
void setDim(unsigned t, unsigned i, Dim d)
Dimension setter.
static void visit(Operation *op, DenseSet< Operation *> &visited)
Visits all the pdl.operand(s), pdl.result(s), and pdl.operation(s) connected to the given operation...
Value buildExp(RewriterBase &rewriter, Location loc, unsigned e, Value v0, Value v1)
Rebuilds SSA format from a tensor expression.
static bool endLoop(Merger &merger, CodeGen &codegen, OpBuilder &builder, linalg::GenericOp op, Operation *loop, unsigned idx, unsigned li, bool needsUniv)
Ends a single loop in current sequence. Returns new values for needsUniv.
bool hasAnyDimOf(const BitVector &bits, Dim d) const
Returns true if any set bit corresponds to queried dim.
This class helps build Operations.
This class provides an abstraction over the different types of ranges over Values.
A class to handle all iteration lattice operations.
Optional< unsigned > buildTensorExpFromLinalg(linalg::GenericOp op)
Builds a tensor expression from the given Linalg operation.
MLIRContext * getContext() const
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
static bool isInvariantAffine(const CodeGen &codegen, AffineExpr a, unsigned ldx, bool &atLevel)
Determines if affine expression is invariant.
Children children
Tensor operations hold the indices of their children.