29 #include <type_traits>
37 if (llvm::all_of(pad, [](int64_t p) {
return p == 0; }))
40 ShapedType inputTy = cast<ShapedType>(input.
getType());
41 Type inputETy = inputTy.getElementType();
42 auto inputShape = inputTy.getShape();
44 assert((inputShape.size() * 2) == pad.size());
49 for (
int i = 0, s = inputShape.size(); i < s; i++) {
50 auto lowPad = pad[i * 2];
51 auto highPad = pad[i * 2 + 1];
52 if (ShapedType::isDynamic(inputShape[i]))
53 paddedShape.push_back(inputShape[i]);
55 paddedShape.push_back(inputShape[i] + highPad + lowPad);
60 Value padValue = rewriter.
create<arith::ConstantOp>(loc, padAttr);
62 return rewriter.
create<tensor::PadOp>(
64 highIndices, padValue);
71 ShapedType resultTy = cast<ShapedType>(conv.
getType());
73 .
create<linalg::GenericOp>(
74 loc, resultTy,
ValueRange({bias, conv}), result, indexingMaps,
77 Value biasVal = args[0];
78 Type resType = args[1].getType();
79 if (resType != biasVal.
getType()) {
80 biasVal = builder.create<arith::ExtSIOp>(loc, resType, biasVal);
82 Value added = builder.create<arith::AddIOp>(loc, biasVal, args[1]);
83 builder.create<linalg::YieldOp>(loc, added);
100 int64_t padBeforeAttr, int64_t padAfterAttr,
101 Value kernelDim, int64_t strideAttr,
102 int64_t dilationAttr,
Type inputETy,
105 auto one = rewriter.
create<arith::ConstantOp>(
108 Value paddedBefore = builder.
create<arith::AddIOp>(inputDim, padBefore);
110 Value paddedAfter = builder.
create<arith::AddIOp>(paddedBefore, padAfter);
112 Value subOne = builder.
create<arith::SubIOp>(kernelDim, one);
114 Value dilated = builder.
create<arith::MulIOp>(dilation, subOne);
115 Value addOne = builder.
create<arith::AddIOp>(dilated, one);
117 Value subtract = builder.
create<arith::SubIOp>(paddedAfter, addOne);
119 Value divide = builder.
create<arith::DivUIOp>(subtract, stride);
120 return builder.
create<arith::AddIOp>(divide, one);
129 ShapedType inputTy = cast<ShapedType>(input.
getType());
130 Type inputETy = inputTy.getElementType();
131 int64_t inputRank = inputTy.getRank();
134 dynDims.resize(resultTy.getRank());
136 for (uint32_t i = 0, s = inputSizeDims.size(); i < s; ++i) {
137 int64_t inputDim = inputSizeDims[i];
138 int64_t kernelDim = kernelSizeDims[i];
139 if (inputTy.isDynamicDim(inputDim)) {
140 auto padTop = padAttr[i * 2];
141 auto padBottom = padAttr[i * 2 + 1];
142 auto stride = strideAttr[i];
143 auto dilation = dilationAttr[i];
144 Value initDynDim = rewriter.
create<tensor::DimOp>(loc, input, inputDim);
146 rewriter.
create<tensor::DimOp>(loc, weight, kernelDim);
150 stride, dilation, inputETy, rewriter);
155 for (
int i = 0; i < inputRank; i++) {
156 if (inputTy.isDynamicDim(i) && !dynDims[i])
157 dynDims[i] = rewriter.
create<tensor::DimOp>(loc, input, i);
169 reassociationMap.resize(outputRank);
170 for (
int i = 0; i < outputRank; i++) {
173 reassociationMap[outputRank - 1].push_back(
179 template <
typename TosaConvOp,
typename LinalgConvOp,
typename LinalgConvQOp>
184 matchAndRewrite(TosaConvOp op,
typename TosaConvOp::Adaptor adaptor,
191 ShapedType inputTy = cast<ShapedType>(input.
getType());
192 ShapedType weightTy = cast<ShapedType>(weight.
getType());
193 ShapedType biasTy = cast<ShapedType>(bias.
getType());
196 Type inputETy = inputTy.getElementType();
197 Type resultETy = resultTy.getElementType();
202 bool isQuantized = op.getQuantizationInfo().has_value();
204 if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
205 return rewriter.notifyMatchFailure(
206 op,
"tosa.conv ops require static shapes for weight and bias");
209 return rewriter.notifyMatchFailure(
210 op,
"tosa.conv ops does not support unsigned integer input");
214 for (
int i = 1; i < resultTy.getRank() - 1; i++) {
215 inputSizeDims.push_back(i);
216 kernelSizeDims.push_back(i);
220 loc, input, weight, resultTy, padAttr.
asArrayRef(),
222 inputSizeDims, kernelSizeDims, rewriter);
224 auto weightShape = weightTy.getShape();
227 TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
229 auto quantizationInfo = *op.getQuantizationInfo();
230 int64_t iZp = quantizationInfo.getInputZp();
239 if (iZp < intMin || iZp > intMax)
240 return rewriter.notifyMatchFailure(
241 op,
"tosa.conv op quantization has zp outside of input range");
243 zeroAttr = rewriter.getIntegerAttr(inputETy, iZp);
248 llvm::append_range(pad, padAttr.
asArrayRef());
249 pad.resize(pad.size() + 2, 0);
250 input =
applyPad(loc, input, pad, zeroAttr, rewriter);
252 if (4 == inputTy.getRank()) {
256 isQuantized ? std::is_same_v<LinalgConvQOp, linalg::Conv2DNhwcHwcfQOp>
257 : std::is_same_v<LinalgConvOp, linalg::Conv2DNhwcHwcfOp>;
264 for (
int i = 1; i < resultTy.getRank(); i++)
265 weightPerm.push_back(i);
266 weightPerm.push_back(0);
269 for (
auto dim : weightPerm)
270 newWeightShape.push_back(weightShape[dim]);
271 auto weightPermAttr = rewriter.getI64TensorAttr(weightPerm);
272 Value weightPermValue =
273 rewriter.create<arith::ConstantOp>(loc, weightPermAttr);
276 weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
284 if (5 == inputTy.getRank()) {
288 for (
int i = 1; i < resultTy.getRank(); i++)
289 weightPerm.push_back(i);
290 weightPerm.push_back(0);
293 for (
auto dim : weightPerm)
294 newWeightShape.push_back(weightShape[dim]);
295 auto weightPermAttr = rewriter.getI64TensorAttr(weightPerm);
296 Value weightPermValue =
297 rewriter.create<arith::ConstantOp>(loc, weightPermAttr);
300 weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
304 auto resultZeroAttr = rewriter.getZeroAttr(resultETy);
305 Value emptyTensor = rewriter.create<tensor::EmptyOp>(
306 loc, resultTy.getShape(), resultETy, filteredDims);
307 Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
308 Value zeroTensor = rewriter
309 .create<linalg::FillOp>(loc,
ValueRange{zero},
318 auto strideAttr = rewriter.getI64TensorAttr(stride);
319 auto dilationAttr = rewriter.getI64TensorAttr(dilation);
324 resultTy.getRank(), 0,
325 {rewriter.getAffineDimExpr(resultTy.getRank() - 1)},
326 rewriter.getContext()));
327 indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultTy.getRank()));
328 indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultTy.getRank()));
330 Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
331 loc, resultTy.getShape(), resultETy, filteredDims);
334 auto quantizationInfo = *op.getQuantizationInfo();
335 auto iZp = rewriter.getI32IntegerAttr(quantizationInfo.getInputZp());
336 auto kZp = rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp());
338 auto iZpVal = rewriter.
create<arith::ConstantOp>(loc, iZp);
339 auto kZpVal = rewriter.
create<arith::ConstantOp>(loc, kZp);
342 .create<LinalgConvQOp>(
343 loc, resultTy,
ValueRange{input, weight, iZpVal, kZpVal},
344 ValueRange{zeroTensor}, strideAttr, dilationAttr)
347 biasEmptyTensor, indexingMaps);
348 rewriter.replaceOp(op, result);
352 Value conv = rewriter
353 .create<LinalgConvOp>(
355 ValueRange{zeroTensor}, strideAttr, dilationAttr)
360 .create<linalg::GenericOp>(
361 loc, resultTy,
ValueRange({bias, conv}), biasEmptyTensor,
365 Value added = nestedBuilder.create<arith::AddFOp>(
366 loc, args[0], args[1]);
367 nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
371 rewriter.replaceOp(op, result);
376 class DepthwiseConvConverter
381 matchAndRewrite(tosa::DepthwiseConv2DOp op, OpAdaptor adaptor,
388 ShapedType inputTy = cast<ShapedType>(input.
getType());
389 ShapedType weightTy = cast<ShapedType>(weight.
getType());
390 ShapedType biasTy = cast<ShapedType>(bias.
getType());
392 int64_t resultRank = resultTy.getRank();
394 Type inputETy = inputTy.getElementType();
395 Type resultETy = resultTy.getElementType();
397 auto padAttr = cast<DenseI64ArrayAttr>(op->
getAttr(
"pad"));
398 auto strideTosaAttr = cast<DenseI64ArrayAttr>(op->
getAttr(
"stride"));
399 auto dilationTosaAttr = cast<DenseI64ArrayAttr>(op->
getAttr(
"dilation"));
401 if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
402 return rewriter.notifyMatchFailure(
403 op,
"tosa.depthwise_conv ops require static shapes");
407 loc, input, weight, resultTy, padAttr.
asArrayRef(),
412 bool isQuantized = op->
hasAttr(
"quantization_info");
416 auto quantizationInfo =
417 cast<tosa::ConvOpQuantizationAttr>(op->
getAttr(
"quantization_info"));
418 iZp = rewriter.getI32IntegerAttr(quantizationInfo.getInputZp());
419 kZp = rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp());
422 auto weightShape = weightTy.getShape();
423 auto resultShape = resultTy.getShape();
426 TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
428 auto quantizationInfo =
429 cast<tosa::ConvOpQuantizationAttr>(op->
getAttr(
"quantization_info"));
430 int64_t iZp = quantizationInfo.getInputZp();
439 if (iZp < intMin || iZp > intMax)
440 return rewriter.notifyMatchFailure(
441 op,
"tosa.depthwise_conv op quantization has zp outside of input "
444 zeroAttr = rewriter.getIntegerAttr(inputETy, iZp);
449 llvm::append_range(pad, padAttr.
asArrayRef());
450 pad.resize(pad.size() + 2, 0);
452 input =
applyPad(loc, input, pad, zeroAttr, rewriter);
459 auto strideAttr = rewriter.getI64TensorAttr(stride);
460 auto dilationAttr = rewriter.getI64TensorAttr(dilation);
461 ShapedType linalgConvTy =
463 weightShape[2], weightShape[3]},
470 {rewriter.getAffineDimExpr(3)}, rewriter.getContext()));
471 indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
472 indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
474 auto resultZeroAttr = rewriter.getZeroAttr(resultETy);
475 Value emptyTensor = rewriter.create<tensor::EmptyOp>(
476 loc, linalgConvTy.getShape(), resultETy, filteredDims);
477 Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
478 Value zeroTensor = rewriter
479 .create<linalg::FillOp>(loc,
ValueRange{zero},
483 Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
484 loc, resultTy.getShape(), resultETy, filteredDims);
486 Value conv = rewriter
487 .create<linalg::DepthwiseConv2DNhwcHwcmOp>(
489 ValueRange{zeroTensor}, strideAttr, dilationAttr)
494 Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
495 loc, resultTy, conv, reassociationMap);
499 .create<linalg::GenericOp>(
500 loc, resultTy,
ValueRange({bias, convReshape}),
501 biasEmptyTensor, indexingMaps,
505 Value added = nestedBuilder.create<arith::AddFOp>(
506 loc, args[0], args[1]);
507 nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
510 rewriter.replaceOp(op, result);
512 auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
513 auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
516 .create<linalg::DepthwiseConv2DNhwcHwcmQOp>(
517 loc, linalgConvTy,
ValueRange{input, weight, iZpVal, kZpVal},
518 ValueRange{zeroTensor}, strideAttr, dilationAttr)
522 Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
523 loc, resultTy, conv, reassociationMap);
525 rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
526 rewriter.replaceOp(op, result);
536 matchAndRewrite(tosa::MatMulOp op, OpAdaptor adaptor,
540 auto outputTy = cast<ShapedType>(op.getType());
541 auto outputElementTy = outputTy.getElementType();
546 if (!outputTy.hasRank() || outputTy.isDynamicDim(0)) {
547 dynDims[0] = rewriter.create<tensor::DimOp>(loc, op->
getOperand(0), 0);
550 if (!outputTy.hasRank() || outputTy.isDynamicDim(1)) {
551 dynDims[1] = rewriter.create<tensor::DimOp>(loc, op->
getOperand(0), 1);
554 if (!outputTy.hasRank() || outputTy.isDynamicDim(2)) {
555 dynDims[2] = rewriter.create<tensor::DimOp>(loc, op->
getOperand(1), 2);
560 auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
561 Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
562 auto emptyTensor = rewriter.create<tensor::EmptyOp>(
563 loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
564 Value zeroTensor = rewriter
565 .create<linalg::FillOp>(loc,
ValueRange{zero},
568 if (!op.getQuantizationInfo()) {
569 rewriter.replaceOpWithNewOp<linalg::BatchMatmulOp>(
575 auto quantizationInfo = *op.getQuantizationInfo();
576 auto aZp = rewriter.
create<arith::ConstantOp>(
577 loc, rewriter.getI32IntegerAttr(quantizationInfo.getAZp()));
578 auto bZp = rewriter.
create<arith::ConstantOp>(
579 loc, rewriter.getI32IntegerAttr(quantizationInfo.getBZp()));
580 rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
582 ValueRange{adaptor.getA(), adaptor.getB(), aZp, bZp}, zeroTensor);
588 class FullyConnectedConverter
593 matchAndRewrite(tosa::FullyConnectedOp op, OpAdaptor adaptor,
596 auto outputTy = cast<ShapedType>(op.getType());
597 auto input = op.getInput();
598 auto inputTy = cast<ShapedType>(input.
getType());
600 auto bias = op.getBias();
602 auto weight = op.getWeight();
603 auto weightTy = cast<ShapedType>(weight.
getType());
604 auto weightShape = weightTy.getShape();
606 auto outputETy = outputTy.getElementType();
611 if (!inputTy.hasRank() || inputTy.isDynamicDim(0)) {
612 dynDims[0] = rewriter.create<tensor::DimOp>(loc, input, 0);
615 if (!weightTy.hasRank() || weightTy.isDynamicDim(0)) {
616 dynDims[1] = rewriter.create<tensor::DimOp>(loc, weight, 0);
626 {rewriter.getAffineDimExpr(1)},
627 rewriter.getContext()));
629 indexingMaps.push_back(rewriter.getMultiDimIdentityMap(outputTy.getRank()));
630 indexingMaps.push_back(rewriter.getMultiDimIdentityMap(outputTy.getRank()));
632 auto emptyTensor = rewriter.create<tensor::EmptyOp>(
633 loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
636 auto resultZeroAttr = rewriter.getZeroAttr(outputETy);
637 Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
638 Value zeroTensor = rewriter
639 .create<linalg::FillOp>(loc,
ValueRange{zero},
644 auto permutationAttr = rewriter.getI64TensorAttr(permutation);
645 Value permutationValue =
646 rewriter.create<arith::ConstantOp>(loc, permutationAttr);
652 Value transposedWeight = rewriter.create<tosa::TransposeOp>(
653 loc, newWeightTy, weight, permutationValue);
655 Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
656 loc, outputTy.getShape(), outputETy, filteredDims);
658 if (!op.getQuantizationInfo()) {
659 Value matmul = rewriter
660 .create<linalg::MatmulOp>(
662 ValueRange{input, transposedWeight}, zeroTensor)
667 .create<linalg::GenericOp>(
668 loc, outputTy,
ValueRange({bias, matmul}), biasEmptyTensor,
672 Value added = nestedBuilder.create<arith::AddFOp>(
673 loc, args[0], args[1]);
674 nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
677 rewriter.replaceOp(op, result);
681 auto quantizationInfo = *op.getQuantizationInfo();
682 auto inputZp = rewriter.
create<arith::ConstantOp>(
683 loc, rewriter.getI32IntegerAttr(quantizationInfo.getInputZp()));
684 auto outputZp = rewriter.
create<arith::ConstantOp>(
685 loc, rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp()));
688 .create<linalg::QuantizedMatmulOp>(
690 ValueRange{input, transposedWeight, inputZp, outputZp},
694 biasEmptyTensor, indexingMaps);
695 rewriter.replaceOp(op, result);
707 Value input = op.getInput();
708 ShapedType inputTy = cast<ShapedType>(input.
getType());
710 ShapedType resultTy = cast<ShapedType>(op.getType());
711 Type resultETy = inputTy.getElementType();
715 if (!dynamicDimsOr.has_value())
720 TypedAttr initialAttr;
722 initialAttr = rewriter.getFloatAttr(
723 resultETy, APFloat::getLargest(
724 cast<FloatType>(resultETy).getFloatSemantics(),
true));
726 if (isa<IntegerType>(resultETy))
727 initialAttr = rewriter.getIntegerAttr(
732 return rewriter.notifyMatchFailure(
733 op,
"Unsupported initial value for tosa.maxpool_2d op");
738 llvm::append_range(pad, op.getPad());
739 pad.resize(pad.size() + 2, 0);
740 Value paddedInput =
applyPad(loc, input, pad, initialAttr, rewriter);
742 Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
747 Attribute strideAttr = rewriter.getI64VectorAttr(stride);
748 Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
751 Value emptyTensor = rewriter.create<tensor::EmptyOp>(
752 loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
754 Value filledEmptyTensor =
756 .create<linalg::FillOp>(loc,
ValueRange{initialValue},
760 Value fakeWindowDims =
761 rewriter.create<tensor::EmptyOp>(loc, kernel, resultETy);
763 rewriter.replaceOpWithNewOp<linalg::PoolingNhwcMaxOp>(
765 filledEmptyTensor, strideAttr, dilationAttr);
777 Value input = op.getInput();
778 ShapedType inputTy = cast<ShapedType>(input.
getType());
779 Type inElementTy = inputTy.getElementType();
781 ShapedType resultTy = cast<ShapedType>(op.getType());
782 Type resultETy = cast<ShapedType>(op.getType()).getElementType();
784 Type accETy = op.getAccType();
785 ShapedType accTy = resultTy.
clone(accETy);
789 if (!dynamicDimsOr.has_value())
796 llvm::append_range(pad, op.getPad());
797 pad.resize(pad.size() + 2, 0);
798 TypedAttr padAttr = rewriter.getZeroAttr(inElementTy);
802 Value paddedInput =
applyPad(loc, input, pad, padAttr, rewriter);
804 auto initialAttr = rewriter.getZeroAttr(accETy);
805 Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
810 Attribute strideAttr = rewriter.getI64VectorAttr(stride);
811 Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
814 Value poolEmptyTensor = rewriter.create<tensor::EmptyOp>(
815 loc, accTy.getShape(), accETy, dynamicDims);
817 Value filledEmptyTensor =
819 .create<linalg::FillOp>(loc,
ValueRange{initialValue},
823 Value fakeWindowDims =
824 rewriter.create<tensor::EmptyOp>(loc, kernel, accETy);
827 Value poolingOp = rewriter
828 .create<linalg::PoolingNhwcSumOp>(
831 filledEmptyTensor, strideAttr, dilationAttr)
836 Value iH = rewriter.create<tensor::DimOp>(loc, poolingOp, 1);
837 Value iW = rewriter.create<tensor::DimOp>(loc, poolingOp, 2);
839 auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
840 iH = rewriter.create<arith::SubIOp>(loc, iH, one);
841 iW = rewriter.create<arith::SubIOp>(loc, iW, one);
843 Value genericEmptyTensor = rewriter.create<tensor::EmptyOp>(
844 loc, resultTy.getShape(), resultETy, dynamicDims);
846 auto affineMap = rewriter.getMultiDimIdentityMap(resultTy.getRank());
847 auto genericOp = rewriter.create<linalg::GenericOp>(
853 auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
861 auto padVal = rewriter.create<arith::ConstantIndexOp>(loc, pad);
862 Value dpos = rewriter.create<arith::SubIOp>(loc, pos, padVal);
864 Value cmp = rewriter.create<arith::CmpIOp>(
865 loc, arith::CmpIPredicate::slt, dpos, zero);
867 rewriter.create<arith::SelectOp>(loc, cmp, dpos, zero);
868 return rewriter.create<arith::AddIOp>(loc, valid, offset)
872 auto coverageFn = [&](int64_t i,
Value isize) ->
Value {
874 rewriter.create<arith::ConstantIndexOp>(loc, stride[i - 1]);
876 rewriter.create<arith::ConstantIndexOp>(loc, kernel[i - 1]);
879 Value left = rewriter.create<linalg::IndexOp>(loc, i);
880 Value right = rewriter.create<arith::SubIOp>(loc, isize, left);
881 left = rewriter.create<arith::MulIOp>(loc, left, strideVal);
882 right = rewriter.create<arith::MulIOp>(loc, right, strideVal);
885 val = padFn(val, left, pad[i * 2]);
886 val = padFn(val, right, pad[i * 2 + 1]);
887 Value cmp = rewriter.create<arith::CmpIOp>(
888 loc, arith::CmpIPredicate::slt, val, one);
889 return rewriter.create<arith::SelectOp>(loc, cmp, one, val);
893 Value kH3 = coverageFn(1, iH);
894 Value kW3 = coverageFn(2, iW);
897 auto count = rewriter.create<arith::IndexCastOp>(
898 loc, rewriter.getI32Type(),
899 rewriter.create<arith::MulIOp>(loc, kH3, kW3));
904 Value poolVal = args[0];
905 if (isa<FloatType>(accETy)) {
906 auto countF = rewriter.create<arith::SIToFPOp>(loc, accETy, count);
907 poolVal = rewriter.create<arith::DivFOp>(loc, poolVal, countF)
913 if (op.getQuantizationInfo()) {
914 auto quantizationInfo = *op.getQuantizationInfo();
915 auto inputZp = rewriter.
create<arith::ConstantOp>(
918 rewriter.create<arith::MulIOp>(loc, accETy, count, inputZp);
920 rewriter.create<arith::SubIOp>(loc, accETy, poolVal, offset);
924 Value one32 = rewriter.create<arith::ConstantOp>(
925 loc, rewriter.getI32IntegerAttr(1));
926 Value thirtyTwo32 = rewriter.create<arith::ConstantOp>(
927 loc, rewriter.getI32IntegerAttr(32));
930 rewriter.create<arith::SubIOp>(loc, count, one32);
932 rewriter.create<math::CountLeadingZerosOp>(loc, countSubOne);
934 rewriter.create<arith::SubIOp>(loc, thirtyTwo32, leadingZeros);
938 rewriter.create<arith::ExtUIOp>(loc, rewriter.getI64Type(), k);
939 Value thirtyShiftPlusOne = rewriter.create<arith::ConstantOp>(
940 loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
942 rewriter.create<arith::ShLIOp>(loc, thirtyShiftPlusOne, k64);
945 Value count64 = rewriter.create<arith::ExtUIOp>(
946 loc, rewriter.getI64Type(), count);
948 rewriter.create<arith::DivUIOp>(loc, numerator, count64);
949 multiplier = rewriter.create<arith::TruncIOp>(
950 loc, rewriter.getI32Type(), multiplier);
954 rewriter.create<arith::TruncIOp>(loc, rewriter.getI8Type(), k);
955 Value thirty8 = rewriter.create<arith::ConstantOp>(
956 loc, rewriter.getI8IntegerAttr(30));
957 Value shift = rewriter.create<arith::AddIOp>(loc, k8, thirty8);
961 .create<tosa::ApplyScaleOp>(loc, rewriter.getI32Type(),
962 poolVal, multiplier, shift,
963 rewriter.getBoolAttr(
false))
968 if (op.getQuantizationInfo()) {
969 auto quantizationInfo = *op.getQuantizationInfo();
970 auto outputZp = rewriter.
create<arith::ConstantOp>(
972 quantizationInfo.getOutputZp()));
973 scaled = rewriter.
create<arith::AddIOp>(loc, scaled, outputZp)
980 auto min = rewriter.create<arith::ConstantIntOp>(
981 loc, APInt::getSignedMinValue(outBitwidth).getSExtValue(),
983 auto max = rewriter.create<arith::ConstantIntOp>(
984 loc, APInt::getSignedMaxValue(outBitwidth).getSExtValue(),
992 rewriter.create<arith::TruncIOp>(loc, resultETy, poolVal);
996 rewriter.create<linalg::YieldOp>(loc, poolVal);
999 rewriter.replaceOp(op, genericOp.getResult(0));
1008 if (
options.preferConv2DKernelLayoutHWCF) {
1009 patterns->
add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcHwcfOp,
1010 linalg::Conv2DNhwcHwcfQOp>>(
1013 patterns->
add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcFhwcOp,
1014 linalg::Conv2DNhwcFhwcQOp>>(
1019 ConvConverter<tosa::Conv3DOp, linalg::Conv3DNdhwcDhwcfOp, linalg::Conv3DNdhwcDhwcfQOp>,
1020 DepthwiseConvConverter,
1024 FullyConnectedConverter>(patterns->
getContext());
static llvm::ManagedStatic< PassManagerOptions > options
static Value clamp(ImplicitLocOpBuilder &builder, Value value, Value lowerBound, Value upperBound)
static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static mlir::Value applyPad(Location loc, Value input, ArrayRef< int64_t > pad, TypedAttr padAttr, OpBuilder &rewriter)
static mlir::Value getConvOutputDim(Location loc, Value inputDim, int64_t padBeforeAttr, int64_t padAfterAttr, Value kernelDim, int64_t strideAttr, int64_t dilationAttr, Type inputETy, OpBuilder &rewriter)
static void createDepthwiseConvCollapseMap(int64_t outputRank, SmallVector< ReassociationExprs, 4 > &reassociationMap, OpBuilder &rewriter)
static mlir::Value linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias, Value conv, Value result, ArrayRef< AffineMap > indexingMaps)
static mlir::Value reifyConstantDim(int64_t attr, ImplicitLocOpBuilder &builder)
static SmallVector< Value > inferDynamicDimsForConv(Location loc, Value input, Value weight, ShapedType resultTy, ArrayRef< int64_t > padAttr, ArrayRef< int64_t > strideAttr, ArrayRef< int64_t > dilationAttr, ArrayRef< int64_t > inputSizeDims, ArrayRef< int64_t > kernelSizeDims, OpBuilder &rewriter)
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
Attributes are known-constant values of operations.
IntegerAttr getIndexAttr(int64_t value)
IntegerAttr getIntegerAttr(Type type, int64_t value)
IntegerAttr getI64IntegerAttr(int64_t value)
AffineExpr getAffineDimExpr(unsigned position)
This class implements a pattern rewriter for use with ConversionPatterns.
ImplicitLocOpBuilder maintains a 'current location', allowing use of the create<> method without spec...
OpTy create(Args &&...args)
Create an operation of specific op type at the current insertion point and location.
void createOrFold(llvm::SmallVectorImpl< Value > &results, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
This class helps build Operations.
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
OpConversionPattern is a wrapper around ConversionPattern that allows for matching and rewriting agai...
Value getOperand(unsigned idx)
Attribute getAttr(StringAttr name)
Return the specified attribute if present, null otherwise.
bool hasAttr(StringAttr name)
Return true if the operation has an attribute with the provided name, false otherwise.
Operation * clone(IRMapping &mapper, CloneOptions options=CloneOptions::all())
Create a deep copy of this operation, remapping any operands that use values outside of the operation...
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Location getLoc()
The source location the operation was defined or derived from.
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
MLIRContext * getContext() const
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
This class provides an abstraction over the various different ranges of value types.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
bool isUnsignedInteger() const
Return true if this is an unsigned integer type (with the specified width).
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
This class provides an abstraction over the different types of ranges over Values.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Type getType() const
Return the type of this value.
Base class for DenseArrayAttr that is instantiated and specialized for each supported element type be...
ArrayRef< T > asArrayRef() const
std::optional< SmallVector< Value > > checkHasDynamicBatchDims(PatternRewriter &rewriter, Op op, ArrayRef< Value > params)
SmallVector< utils::IteratorType > getNParallelLoopsAttrs(unsigned nParallelLoops)
SmallVector< Value > condenseValues(const SmallVector< Value > &values)
Value clampIntHelper(Location loc, Value arg, Value min, Value max, OpBuilder &rewriter)
void populateTosaToLinalgNamedConversionPatterns(RewritePatternSet *patterns, const TosaToLinalgNamedOptions &options)
Populates conversion passes from TOSA dialect to Linalg named operations.
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
This class represents an efficient way to signal success or failure.
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...