32 #include <type_traits>
40 if (llvm::all_of(pad, [](int64_t p) {
return p == 0; }))
43 ShapedType inputTy = cast<ShapedType>(input.
getType());
44 Type inputETy = inputTy.getElementType();
45 auto inputShape = inputTy.getShape();
47 assert((inputShape.size() * 2) == pad.size());
52 for (
size_t i : llvm::seq(inputShape.size())) {
53 auto lowPad = pad[i * 2];
54 auto highPad = pad[i * 2 + 1];
55 if (ShapedType::isDynamic(inputShape[i]))
56 paddedShape.push_back(inputShape[i]);
58 paddedShape.push_back(inputShape[i] + highPad + lowPad);
63 Value padValue = rewriter.
create<arith::ConstantOp>(loc, padAttr);
65 return rewriter.
create<tensor::PadOp>(
67 highIndices, padValue);
74 ShapedType resultTy = cast<ShapedType>(conv.
getType());
76 .
create<linalg::GenericOp>(
77 loc, resultTy,
ValueRange({bias, conv}), result, indexingMaps,
80 Value biasVal = args[0];
81 Type resType = args[1].getType();
82 if (resType != biasVal.
getType()) {
83 biasVal = builder.create<arith::ExtSIOp>(loc, resType, biasVal);
85 Value added = builder.create<arith::AddIOp>(loc, biasVal, args[1]);
86 builder.create<linalg::YieldOp>(loc, added);
95 ShapedType resultTy = cast<ShapedType>(result.
getType());
96 ShapedType sourceTy = cast<ShapedType>(source.
getType());
97 const int64_t resultRank = resultTy.getRank();
98 const int64_t sourceRank = sourceTy.getRank();
106 assert(sourceTy.hasStaticShape() &&
107 "Dynamic broadcasting shapes not supported!");
108 if (sourceRank == 1 && sourceTy.getDimSize(0) == 1) {
111 for (
auto dim : llvm::seq<int64_t>(0, sourceRank)) {
113 sourceDims.push_back(expr);
126 ShapedType resultTy = cast<ShapedType>(result.
getType());
127 const int64_t resultRank = resultTy.getRank();
135 .
create<linalg::GenericOp>(
136 loc, resultTy,
ValueRange({source}), result, indexingMaps,
139 Value biasVal = args[0];
140 Type resType = args[1].getType();
141 if (resType != biasVal.
getType()) {
142 biasVal = builder.create<arith::ExtSIOp>(loc, resType, biasVal);
144 builder.create<linalg::YieldOp>(loc, biasVal);
151 return builder.
create<arith::ConstantIndexOp>(attr);
159 int64_t padBeforeAttr,
160 int64_t padAfterAttr,
Value kernelDim,
162 int64_t dilationAttr,
165 auto one = rewriter.
create<arith::ConstantOp>(
168 Value paddedBefore = builder.
create<arith::AddIOp>(inputDim, padBefore);
170 Value paddedAfter = builder.
create<arith::AddIOp>(paddedBefore, padAfter);
172 Value subOne = builder.
create<arith::SubIOp>(kernelDim, one);
174 Value dilated = builder.
create<arith::MulIOp>(dilation, subOne);
175 Value addOne = builder.
create<arith::AddIOp>(dilated, one);
177 Value subtract = builder.
create<arith::SubIOp>(paddedAfter, addOne);
179 Value divide = builder.
create<arith::DivUIOp>(subtract, stride);
180 return builder.
create<arith::AddIOp>(divide, one);
189 ShapedType inputTy = cast<ShapedType>(input.
getType());
190 int64_t inputRank = inputTy.getRank();
193 dynDims.resize(resultTy.getRank());
195 for (uint32_t i = 0, s = inputSizeDims.size(); i < s; ++i) {
196 int64_t inputDim = inputSizeDims[i];
197 int64_t kernelDim = kernelSizeDims[i];
198 if (resultTy.isDynamicDim(inputDim)) {
199 auto padTop = padAttr[i * 2];
200 auto padBottom = padAttr[i * 2 + 1];
201 auto stride = strideAttr[i];
202 auto dilation = dilationAttr[i];
203 Value initDynDim = rewriter.
create<tensor::DimOp>(loc, input, inputDim);
205 rewriter.
create<tensor::DimOp>(loc, weight, kernelDim);
209 kernelDynDim, stride, dilation, rewriter);
214 for (
int i = 0; i < inputRank; i++) {
215 if (resultTy.isDynamicDim(i) && !dynDims[i])
216 dynDims[i] = rewriter.
create<tensor::DimOp>(loc, input, i);
228 reassociationMap.resize(outputRank);
229 for (
int i = 0; i < outputRank; i++) {
232 reassociationMap[outputRank - 1].push_back(
238 template <
typename TosaConvOp,
typename LinalgConvOp,
typename LinalgConvQOp>
243 matchAndRewrite(TosaConvOp op,
typename TosaConvOp::Adaptor adaptor,
246 Value input = op->getOperand(0);
247 Value weight = op->getOperand(1);
248 Value bias = op->getOperand(2);
250 ShapedType inputTy = cast<ShapedType>(input.
getType());
251 ShapedType weightTy = cast<ShapedType>(weight.
getType());
252 ShapedType biasTy = cast<ShapedType>(bias.
getType());
253 ShapedType resultTy = cast<ShapedType>(op->getResult(0).getType());
255 Type inputETy = inputTy.getElementType();
256 Type resultETy = resultTy.getElementType();
263 if (llvm::failed(failureOrMaybeZps))
266 auto maybeZps = failureOrMaybeZps.value();
268 if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
269 return rewriter.notifyMatchFailure(
270 op,
"tosa.conv ops require static shapes for weight and bias");
273 return rewriter.notifyMatchFailure(
274 op,
"tosa.conv ops does not support unsigned integer input");
278 for (
int i = 1; i < resultTy.getRank() - 1; i++) {
279 inputSizeDims.push_back(i);
280 kernelSizeDims.push_back(i);
284 loc, input, weight, resultTy, padAttr.
asArrayRef(),
286 inputSizeDims, kernelSizeDims, rewriter);
288 auto weightShape = weightTy.getShape();
291 TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
300 if (maybeZps->inputZp < intMin || maybeZps->inputZp > intMax)
301 return rewriter.notifyMatchFailure(
302 op,
"tosa.conv op quantization has zp outside of input range");
304 zeroAttr = rewriter.getIntegerAttr(inputETy, maybeZps->inputZp);
309 llvm::append_range(pad, padAttr.
asArrayRef());
310 pad.resize(pad.size() + 2, 0);
311 input =
applyPad(loc, input, pad, zeroAttr, rewriter);
313 if (4 == inputTy.getRank()) {
317 maybeZps ? std::is_same_v<LinalgConvQOp, linalg::Conv2DNhwcHwcfQOp>
318 : std::is_same_v<LinalgConvOp, linalg::Conv2DNhwcHwcfOp>;
325 for (
int i = 1; i < resultTy.getRank(); i++)
326 weightPerm.push_back(i);
327 weightPerm.push_back(0);
330 for (
auto dim : weightPerm)
331 newWeightShape.push_back(weightShape[dim]);
332 auto weightPermAttr = rewriter.getI32TensorAttr(weightPerm);
333 Value weightPermValue =
334 rewriter.create<arith::ConstantOp>(loc, weightPermAttr);
337 weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
345 if (5 == inputTy.getRank()) {
349 for (
int i = 1; i < resultTy.getRank(); i++)
350 weightPerm.push_back(i);
351 weightPerm.push_back(0);
354 for (
auto dim : weightPerm)
355 newWeightShape.push_back(weightShape[dim]);
356 auto weightPermAttr = rewriter.getI32TensorAttr(weightPerm);
357 Value weightPermValue =
358 rewriter.create<arith::ConstantOp>(loc, weightPermAttr);
361 weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
370 auto strideAttr = rewriter.getI64TensorAttr(stride);
371 auto dilationAttr = rewriter.getI64TensorAttr(dilation);
373 Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
374 loc, resultTy.getShape(), resultETy, filteredDims);
376 Value broadcastBias =
380 auto iZp = rewriter.getI32IntegerAttr(maybeZps->inputZp);
381 auto kZp = rewriter.getI32IntegerAttr(maybeZps->weightZp);
383 auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
384 auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
388 .create<LinalgConvQOp>(
389 loc, resultTy,
ValueRange{input, weight, iZpVal, kZpVal},
390 ValueRange{broadcastBias}, strideAttr, dilationAttr)
393 rewriter.replaceOp(op, conv);
397 Value conv = rewriter
398 .create<LinalgConvOp>(
400 ValueRange{broadcastBias}, strideAttr, dilationAttr)
403 rewriter.replaceOp(op, conv);
408 class DepthwiseConvConverter
413 matchAndRewrite(tosa::DepthwiseConv2DOp op, OpAdaptor adaptor,
416 Value input = op->getOperand(0);
417 Value weight = op->getOperand(1);
418 Value bias = op->getOperand(2);
420 ShapedType inputTy = cast<ShapedType>(input.
getType());
421 ShapedType weightTy = cast<ShapedType>(weight.
getType());
422 ShapedType biasTy = cast<ShapedType>(bias.
getType());
423 ShapedType resultTy = cast<ShapedType>(op->getResult(0).getType());
424 int64_t resultRank = resultTy.getRank();
426 Type inputETy = inputTy.getElementType();
427 Type resultETy = resultTy.getElementType();
429 auto padAttr = cast<DenseI64ArrayAttr>(op->getAttr(
"pad"));
430 auto strideTosaAttr = cast<DenseI64ArrayAttr>(op->getAttr(
"stride"));
431 auto dilationTosaAttr = cast<DenseI64ArrayAttr>(op->getAttr(
"dilation"));
433 if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
434 return rewriter.notifyMatchFailure(
435 op,
"tosa.depthwise_conv ops require static shapes");
439 loc, input, weight, resultTy, padAttr.
asArrayRef(),
445 if (llvm::failed(failureOrMaybeZps))
448 auto maybeZps = failureOrMaybeZps.value();
450 auto weightShape = weightTy.getShape();
451 auto resultShape = resultTy.getShape();
454 TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
463 if (maybeZps->inputZp < intMin || maybeZps->inputZp > intMax)
464 return rewriter.notifyMatchFailure(
465 op,
"tosa.depthwise_conv op quantization has zp outside of input "
468 zeroAttr = rewriter.getIntegerAttr(inputETy, maybeZps->inputZp);
473 llvm::append_range(pad, padAttr.
asArrayRef());
474 pad.resize(pad.size() + 2, 0);
476 input =
applyPad(loc, input, pad, zeroAttr, rewriter);
483 auto strideAttr = rewriter.getI64TensorAttr(stride);
484 auto dilationAttr = rewriter.getI64TensorAttr(dilation);
485 ShapedType linalgConvTy =
487 weightShape[2], weightShape[3]},
490 auto resultZeroAttr = rewriter.getZeroAttr(resultETy);
491 Value emptyTensor = rewriter.create<tensor::EmptyOp>(
492 loc, linalgConvTy.getShape(), resultETy, filteredDims);
493 Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
494 Value zeroTensor = rewriter
495 .create<linalg::FillOp>(loc,
ValueRange{zero},
499 Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
500 loc, resultTy.getShape(), resultETy, filteredDims);
505 indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
506 indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
509 Value conv = rewriter
510 .create<linalg::DepthwiseConv2DNhwcHwcmOp>(
512 ValueRange{zeroTensor}, strideAttr, dilationAttr)
517 Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
518 loc, resultTy, conv, reassociationMap);
522 .create<linalg::GenericOp>(
523 loc, resultTy,
ValueRange({bias, convReshape}),
524 biasEmptyTensor, indexingMaps,
528 Value added = nestedBuilder.create<arith::AddFOp>(
529 loc, args[0], args[1]);
530 nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
533 rewriter.replaceOp(op, result);
535 IntegerAttr iZp = rewriter.getI32IntegerAttr(maybeZps->inputZp);
536 IntegerAttr wZp = rewriter.getI32IntegerAttr(maybeZps->weightZp);
537 auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
538 auto kZpVal = rewriter.create<arith::ConstantOp>(loc, wZp);
541 .create<linalg::DepthwiseConv2DNhwcHwcmQOp>(
542 loc, linalgConvTy,
ValueRange{input, weight, iZpVal, kZpVal},
543 ValueRange{zeroTensor}, strideAttr, dilationAttr)
547 Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
548 loc, resultTy, conv, reassociationMap);
550 rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
551 rewriter.replaceOp(op, result);
561 matchAndRewrite(tosa::MatMulOp op, OpAdaptor adaptor,
565 auto outputTy = cast<ShapedType>(op.getType());
566 auto outputElementTy = outputTy.getElementType();
569 dynDims.resize(cast<ShapedType>(op->getResult(0).getType()).getRank());
571 if (!outputTy.hasRank() || outputTy.isDynamicDim(0)) {
572 dynDims[0] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 0);
575 if (!outputTy.hasRank() || outputTy.isDynamicDim(1)) {
576 dynDims[1] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 1);
579 if (!outputTy.hasRank() || outputTy.isDynamicDim(2)) {
580 dynDims[2] = rewriter.create<tensor::DimOp>(loc, op->getOperand(1), 2);
585 auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
586 Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
587 auto emptyTensor = rewriter.create<tensor::EmptyOp>(
588 loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
589 Value zeroTensor = rewriter
590 .create<linalg::FillOp>(loc,
ValueRange{zero},
593 if (!op.getAZp() && !op.getBZp()) {
594 rewriter.replaceOpWithNewOp<linalg::BatchMatmulOp>(
600 auto aZp = rewriter.create<arith::ConstantOp>(loc, op.getAZpAttr());
601 auto bZp = rewriter.create<arith::ConstantOp>(loc, op.getBZpAttr());
602 rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
604 ValueRange{adaptor.getA(), adaptor.getB(), aZp, bZp}, zeroTensor);
610 class FullyConnectedConverter
615 matchAndRewrite(tosa::FullyConnectedOp op, OpAdaptor adaptor,
618 auto outputTy = cast<ShapedType>(op.getType());
619 auto input = op.getInput();
620 auto inputTy = cast<ShapedType>(input.
getType());
622 auto bias = op.getBias();
624 auto weight = op.getWeight();
625 auto weightTy = cast<ShapedType>(weight.
getType());
626 auto weightShape = weightTy.getShape();
628 auto outputETy = outputTy.getElementType();
631 dynDims.resize(cast<ShapedType>(op->getResult(0).getType()).getRank());
633 if (!inputTy.hasRank() || inputTy.isDynamicDim(0)) {
634 dynDims[0] = rewriter.create<tensor::DimOp>(loc, input, 0);
637 if (!weightTy.hasRank() || weightTy.isDynamicDim(0)) {
638 dynDims[1] = rewriter.create<tensor::DimOp>(loc, weight, 0);
644 auto permutationAttr = rewriter.getI64TensorAttr(permutation);
645 Value permutationValue =
646 rewriter.create<arith::ConstantOp>(loc, permutationAttr);
652 Value transposedWeight = rewriter.create<tosa::TransposeOp>(
653 loc, newWeightTy, weight, permutationValue);
655 Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
656 loc, outputTy.getShape(), outputETy, filteredDims);
658 Value broadcastBias =
661 if (!op.getInputZp() && !op.getWeightZp()) {
662 Value matmul = rewriter
663 .create<linalg::MatmulOp>(
665 ValueRange{input, transposedWeight}, broadcastBias)
668 rewriter.replaceOp(op, matmul);
672 auto inputZp = rewriter.create<arith::ConstantOp>(loc, op.getInputZpAttr());
674 rewriter.create<arith::ConstantOp>(loc, op.getWeightZpAttr());
677 .create<linalg::QuantizedMatmulOp>(
679 ValueRange{input, transposedWeight, inputZp, outputZp},
683 rewriter.replaceOp(op, matmul);
694 computeDynamicOutputSizes(tosa::MaxPool2dOp op, OpAdaptor adaptor,
699 Value input = adaptor.getInput();
707 if (resultTy.isDynamicDim(0))
708 dynamicDims.push_back(rewriter.
create<tensor::DimOp>(loc, input, 0));
711 for (int64_t dim : {1, 2}) {
712 if (!resultTy.isDynamicDim(dim))
716 int64_t index = dim - 1;
719 Value ihw = rewriter.
create<tensor::DimOp>(loc, input, dim);
722 Value khw = rewriter.
create<arith::ConstantIndexOp>(loc, kernel[index]);
726 pad[index * 2 + 1], khw, stride[index],
728 dynamicDims.push_back(ohw);
732 if (resultTy.isDynamicDim(3))
733 dynamicDims.push_back(rewriter.
create<tensor::DimOp>(loc, input, 3));
739 matchAndRewrite(tosa::MaxPool2dOp op, OpAdaptor adaptor,
742 Value input = adaptor.getInput();
743 ShapedType inputTy = cast<ShapedType>(input.
getType());
745 bool isUnsigned = op.getType().getElementType().isUnsignedInteger();
746 ShapedType resultTy =
747 cast<ShapedType>(getTypeConverter()->convertType(op.getType()));
750 Type resultETy = inputTy.getElementType();
753 computeDynamicOutputSizes(op, adaptor, rewriter);
756 TypedAttr initialAttr;
759 resultETy, APFloat::getLargest(
760 cast<FloatType>(resultETy).getFloatSemantics(),
true));
765 else if (isa<IntegerType>(resultETy))
772 op,
"Unsupported initial value for tosa.maxpool_2d op");
777 llvm::append_range(pad, op.getPad());
778 pad.resize(pad.size() + 2, 0);
780 Value paddedInput =
applyPad(loc, input, pad, initialAttr, rewriter);
782 Value initialValue = rewriter.
create<arith::ConstantOp>(loc, initialAttr);
791 Value emptyTensor = rewriter.
create<tensor::EmptyOp>(
792 loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
794 Value filledEmptyTensor =
795 rewriter.
create<linalg::FillOp>(loc, initialValue, emptyTensor)
798 Value fakeWindowDims =
799 rewriter.
create<tensor::EmptyOp>(loc, kernel, resultETy);
804 filledEmptyTensor, strideAttr, dilationAttr);
808 filledEmptyTensor, strideAttr, dilationAttr);
818 LogicalResult matchAndRewrite(tosa::AvgPool2dOp op,
821 Value input = op.getInput();
822 ShapedType inputTy = cast<ShapedType>(input.
getType());
823 Type inElementTy = inputTy.getElementType();
825 ShapedType resultTy = cast<ShapedType>(op.getType());
826 Type resultETy = cast<ShapedType>(op.getType()).getElementType();
828 Type accETy = op.getAccType();
829 ShapedType accTy = resultTy.clone(accETy);
833 if (!dynamicDimsOr.has_value())
840 llvm::append_range(pad, op.getPad());
841 pad.resize(pad.size() + 2, 0);
842 TypedAttr padAttr = rewriter.
getZeroAttr(inElementTy);
846 Value paddedInput =
applyPad(loc, input, pad, padAttr, rewriter);
849 Value initialValue = rewriter.
create<arith::ConstantOp>(loc, initialAttr);
858 Value poolEmptyTensor = rewriter.
create<tensor::EmptyOp>(
859 loc, accTy.getShape(), accETy, dynamicDims);
861 Value filledEmptyTensor =
867 Value fakeWindowDims =
868 rewriter.
create<tensor::EmptyOp>(loc, kernel, accETy);
871 Value poolingOp = rewriter
872 .
create<linalg::PoolingNhwcSumOp>(
875 filledEmptyTensor, strideAttr, dilationAttr)
880 Value iH = rewriter.
create<tensor::DimOp>(loc, poolingOp, 1);
881 Value iW = rewriter.
create<tensor::DimOp>(loc, poolingOp, 2);
883 auto one = rewriter.
create<arith::ConstantIndexOp>(loc, 1);
884 iH = rewriter.
create<arith::SubIOp>(loc, iH, one);
885 iW = rewriter.
create<arith::SubIOp>(loc, iW, one);
887 Value genericEmptyTensor = rewriter.
create<tensor::EmptyOp>(
888 loc, resultTy.getShape(), resultETy, dynamicDims);
891 auto genericOp = rewriter.
create<linalg::GenericOp>(
897 auto zero = rewriter.
create<arith::ConstantIndexOp>(loc, 0);
905 auto padVal = rewriter.
create<arith::ConstantIndexOp>(loc, pad);
906 Value dpos = rewriter.
create<arith::SubIOp>(loc, pos, padVal);
908 Value offset = rewriter.
create<arith::MinSIOp>(loc, dpos, zero);
909 return rewriter.
create<arith::AddIOp>(loc, valid, offset)
913 auto coverageFn = [&](int64_t i,
Value isize) ->
Value {
915 rewriter.
create<arith::ConstantIndexOp>(loc, stride[i - 1]);
917 rewriter.
create<arith::ConstantIndexOp>(loc, kernel[i - 1]);
920 Value left = rewriter.
create<linalg::IndexOp>(loc, i);
921 Value right = rewriter.
create<arith::SubIOp>(loc, isize, left);
922 left = rewriter.
create<arith::MulIOp>(loc, left, strideVal);
923 right = rewriter.
create<arith::MulIOp>(loc, right, strideVal);
926 val = padFn(val, left, pad[i * 2]);
927 val = padFn(val, right, pad[i * 2 + 1]);
928 return rewriter.
create<arith::MaxSIOp>(loc, one, val);
932 Value kH3 = coverageFn(1, iH);
933 Value kW3 = coverageFn(2, iW);
936 auto count = rewriter.
create<arith::IndexCastOp>(
938 rewriter.
create<arith::MulIOp>(loc, kH3, kW3));
943 Value poolVal = args[0];
944 if (isa<FloatType>(accETy)) {
945 auto countF = rewriter.
create<arith::SIToFPOp>(loc, accETy, count);
946 poolVal = rewriter.
create<arith::DivFOp>(loc, poolVal, countF)
948 if (accETy.getIntOrFloatBitWidth() >
951 rewriter.
create<arith::TruncFOp>(loc, resultETy, poolVal);
956 if (op.getInputZp()) {
958 rewriter.
create<arith::ConstantOp>(loc, op.getInputZpAttr());
960 rewriter.
create<arith::MulIOp>(loc, accETy, count, inputZp);
962 rewriter.
create<arith::SubIOp>(loc, accETy, poolVal, offset);
968 Value thirtyTwo32 = rewriter.
create<arith::ConstantOp>(
972 rewriter.
create<arith::SubIOp>(loc, count, one32);
974 rewriter.
create<math::CountLeadingZerosOp>(loc, countSubOne);
976 rewriter.
create<arith::SubIOp>(loc, thirtyTwo32, leadingZeros);
981 Value thirtyShiftPlusOne = rewriter.
create<arith::ConstantOp>(
984 rewriter.
create<arith::ShLIOp>(loc, thirtyShiftPlusOne, k64);
990 rewriter.
create<arith::DivUIOp>(loc, numerator, count64);
991 multiplier = rewriter.
create<arith::TruncIOp>(
997 Value thirty8 = rewriter.
create<arith::ConstantOp>(
999 Value shift = rewriter.
create<arith::AddIOp>(loc, k8, thirty8);
1004 poolVal, multiplier, shift,
1010 if (op.getOutputZp()) {
1012 rewriter.
create<arith::ConstantOp>(loc, op.getOutputZpAttr());
1013 scaled = rewriter.
create<arith::AddIOp>(loc, scaled, outputZp)
1020 auto min = rewriter.
create<arith::ConstantIntOp>(
1021 loc, APInt::getSignedMinValue(outBitwidth).getSExtValue(),
1023 auto max = rewriter.
create<arith::ConstantIntOp>(
1024 loc, APInt::getSignedMaxValue(outBitwidth).getSExtValue(),
1033 rewriter.
create<arith::TruncIOp>(loc, resultETy, poolVal);
1037 rewriter.
create<linalg::YieldOp>(loc, poolVal);
1040 rewriter.
replaceOp(op, genericOp.getResult(0));
1049 LogicalResult matchAndRewrite(tosa::TransposeOp op,
1052 if (failed(op.getConstantPerms(constantPerms)))
1061 auto permutedSizes =
1062 applyTOSAPermutation<OpFoldResult>(inputSizes, constantPerms);
1064 auto permutedInit = rewriter.
create<tensor::EmptyOp>(
1065 loc, permutedSizes, op.getInput1().getType().getElementType());
1067 op, op.getInput1(), permutedInit,
1068 llvm::to_vector(llvm::map_range(
1069 constantPerms, [](int32_t v) -> int64_t {
return v; })));
1077 const TosaToLinalgNamedOptions &
options) {
1078 if (
options.preferConv2DKernelLayoutHWCF) {
1079 patterns->add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcHwcfOp,
1080 linalg::Conv2DNhwcHwcfQOp>>(
1083 patterns->add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcFhwcOp,
1084 linalg::Conv2DNhwcFhwcQOp>>(
1089 ConvConverter<tosa::Conv3DOp, linalg::Conv3DNdhwcDhwcfOp, linalg::Conv3DNdhwcDhwcfQOp>,
1090 DepthwiseConvConverter,
1093 FullyConnectedConverter,
1099 >(converter,
patterns->getContext());
static Value getZero(OpBuilder &b, Location loc, Type elementType)
Get zero value for an element type.
static llvm::ManagedStatic< PassManagerOptions > options
static Value clamp(ImplicitLocOpBuilder &builder, Value value, Value lowerBound, Value upperBound)
static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static AffineMap getBroadcastingMap(PatternRewriter &rewriter, Value source, Value result)
static mlir::Value applyPad(Location loc, Value input, ArrayRef< int64_t > pad, TypedAttr padAttr, OpBuilder &rewriter)
static mlir::Value linalgBroadcastAndMaybeExtSI(PatternRewriter &rewriter, Location loc, Value source, Value result)
static void createDepthwiseConvCollapseMap(int64_t outputRank, SmallVector< ReassociationExprs, 4 > &reassociationMap, OpBuilder &rewriter)
static mlir::Value linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias, Value conv, Value result, ArrayRef< AffineMap > indexingMaps)
static mlir::Value getConvOrPoolOutputDim(Location loc, Value inputDim, int64_t padBeforeAttr, int64_t padAfterAttr, Value kernelDim, int64_t strideAttr, int64_t dilationAttr, OpBuilder &rewriter)
static mlir::Value reifyConstantDim(int64_t attr, ImplicitLocOpBuilder &builder)
static SmallVector< Value > inferDynamicDimsForConv(Location loc, Value input, Value weight, ShapedType resultTy, ArrayRef< int64_t > padAttr, ArrayRef< int64_t > strideAttr, ArrayRef< int64_t > dilationAttr, ArrayRef< int64_t > inputSizeDims, ArrayRef< int64_t > kernelSizeDims, OpBuilder &rewriter)
A multi-dimensional affine map Affine map's are immutable like Type's, and they are uniqued.
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
Attributes are known-constant values of operations.
IntegerAttr getIndexAttr(int64_t value)
IntegerAttr getI32IntegerAttr(int32_t value)
IntegerAttr getIntegerAttr(Type type, int64_t value)
AffineMap getMultiDimIdentityMap(unsigned rank)
FloatAttr getFloatAttr(Type type, double value)
AffineExpr getAffineConstantExpr(int64_t constant)
IntegerAttr getI64IntegerAttr(int64_t value)
BoolAttr getBoolAttr(bool value)
TypedAttr getZeroAttr(Type type)
AffineExpr getAffineDimExpr(unsigned position)
MLIRContext * getContext() const
DenseIntElementsAttr getI64VectorAttr(ArrayRef< int64_t > values)
IntegerAttr getI8IntegerAttr(int8_t value)
This class implements a pattern rewriter for use with ConversionPatterns.
void replaceOp(Operation *op, ValueRange newValues) override
Replace the given operation with the new values.
ImplicitLocOpBuilder maintains a 'current location', allowing use of the create<> method without spec...
OpTy create(Args &&...args)
Create an operation of specific op type at the current insertion point and location.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
This class helps build Operations.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
OpConversionPattern is a wrapper around ConversionPattern that allows for matching and rewriting agai...
OpConversionPattern(MLIRContext *context, PatternBenefit benefit=1)
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
std::enable_if_t<!std::is_convertible< CallbackT, Twine >::value, LogicalResult > notifyMatchFailure(Location loc, CallbackT &&reasonCallback)
Used to notify the listener that the IR failed to be rewritten because of a match failure,...
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replace the results of the given (original) op with a new op that is created without verification (re...
Tensor types represent multi-dimensional arrays, and have two variants: RankedTensorType and Unranked...
This class provides an abstraction over the various different ranges of value types.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
bool isUnsignedInteger() const
Return true if this is an unsigned integer type (with the specified width).
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Base class for DenseArrayAttr that is instantiated and specialized for each supported element type be...
ArrayRef< T > asArrayRef() const
SmallVector< OpFoldResult > getMixedSizes(OpBuilder &builder, Location loc, Value value)
Return the dimensions of the given tensor value.
std::optional< SmallVector< Value > > checkHasDynamicBatchDims(PatternRewriter &rewriter, Op op, ArrayRef< Value > params)
std::enable_if_t< is_tosa_conv_v< TosaConvOp >, FailOrMaybeZP > extractConvZpPair(TosaConvOp op, PatternRewriter &rewriter)
SmallVector< utils::IteratorType > getNParallelLoopsAttrs(unsigned nParallelLoops)
SmallVector< Value > condenseValues(const SmallVector< Value > &values)
Value clampIntHelper(Location loc, Value arg, Value min, Value max, OpBuilder &rewriter, bool isUnsigned)
void populateTosaToLinalgNamedConversionPatterns(const TypeConverter &converter, RewritePatternSet *patterns, const TosaToLinalgNamedOptions &options)
Populates conversion passes from TOSA dialect to Linalg named operations.
Include the generated interface declarations.
const FrozenRewritePatternSet & patterns
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...