MLIR  18.0.0git
TosaToLinalgNamed.cpp
Go to the documentation of this file.
1 //===- TosaToLinalgNamed.cpp - Lowering Tosa to Linalg Named Ops ----------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // These rewriters lower from the Tosa to the Linalg named ops.
10 //
11 //===----------------------------------------------------------------------===//
12 
23 #include "mlir/IR/Matchers.h"
24 #include "mlir/IR/PatternMatch.h"
27 
28 #include <numeric>
29 #include <type_traits>
30 
31 using namespace mlir;
32 using namespace mlir::tosa;
33 
35  TypedAttr padAttr, OpBuilder &rewriter) {
36  // Input should be padded if necessary.
37  if (llvm::all_of(pad, [](int64_t p) { return p == 0; }))
38  return input;
39 
40  ShapedType inputTy = cast<ShapedType>(input.getType());
41  Type inputETy = inputTy.getElementType();
42  auto inputShape = inputTy.getShape();
43 
44  assert((inputShape.size() * 2) == pad.size());
45 
46  SmallVector<int64_t, 4> paddedShape;
48  SmallVector<OpFoldResult, 8> highIndices;
49  for (int i = 0, s = inputShape.size(); i < s; i++) {
50  auto lowPad = pad[i * 2];
51  auto highPad = pad[i * 2 + 1];
52  if (ShapedType::isDynamic(inputShape[i]))
53  paddedShape.push_back(inputShape[i]);
54  else
55  paddedShape.push_back(inputShape[i] + highPad + lowPad);
56  lowIndices.push_back(rewriter.getIndexAttr(lowPad));
57  highIndices.push_back(rewriter.getIndexAttr(highPad));
58  }
59 
60  Value padValue = rewriter.create<arith::ConstantOp>(loc, padAttr);
61 
62  return rewriter.create<tensor::PadOp>(
63  loc, RankedTensorType::get(paddedShape, inputETy), input, lowIndices,
64  highIndices, padValue);
65 }
66 
67 static mlir::Value
69  Value conv, Value result,
70  ArrayRef<AffineMap> indexingMaps) {
71  ShapedType resultTy = cast<ShapedType>(conv.getType());
72  return rewriter
73  .create<linalg::GenericOp>(
74  loc, resultTy, ValueRange({bias, conv}), result, indexingMaps,
75  getNParallelLoopsAttrs(resultTy.getRank()),
76  [](OpBuilder &builder, Location loc, ValueRange args) {
77  Value biasVal = args[0];
78  Type resType = args[1].getType();
79  if (resType != biasVal.getType()) {
80  biasVal = builder.create<arith::ExtSIOp>(loc, resType, biasVal);
81  }
82  Value added = builder.create<arith::AddIOp>(loc, biasVal, args[1]);
83  builder.create<linalg::YieldOp>(loc, added);
84  })
85  .getResult(0);
86 }
87 
88 static mlir::Value reifyConstantDim(int64_t attr,
89  ImplicitLocOpBuilder &builder) {
90  return builder.createOrFold<arith::IndexCastOp>(
91  builder.getIndexType(),
92  builder.create<arith::ConstantOp>(builder.getI64IntegerAttr(attr)));
93 }
94 
95 // Calculating the output width/height using the formula:
96 // H = ((IH+pad_top+pad_bottom-(dilation_y*(KH-1)+1))/stride_y)+1
97 // W = ((IW+pad_left+pad_right-(dilation_x*(KW-1)+1))/stride_x)+1
98 
100  int64_t padBeforeAttr, int64_t padAfterAttr,
101  Value kernelDim, int64_t strideAttr,
102  int64_t dilationAttr, Type inputETy,
103  OpBuilder &rewriter) {
104  ImplicitLocOpBuilder builder(loc, rewriter);
105  auto one = rewriter.create<arith::ConstantOp>(
106  loc, IntegerAttr::get(inputDim.getType(), 1));
107  Value padBefore = reifyConstantDim(padBeforeAttr, builder);
108  Value paddedBefore = builder.create<arith::AddIOp>(inputDim, padBefore);
109  Value padAfter = reifyConstantDim(padAfterAttr, builder);
110  Value paddedAfter = builder.create<arith::AddIOp>(paddedBefore, padAfter);
111 
112  Value subOne = builder.create<arith::SubIOp>(kernelDim, one);
113  Value dilation = reifyConstantDim(dilationAttr, builder);
114  Value dilated = builder.create<arith::MulIOp>(dilation, subOne);
115  Value addOne = builder.create<arith::AddIOp>(dilated, one);
116 
117  Value subtract = builder.create<arith::SubIOp>(paddedAfter, addOne);
118  Value stride = reifyConstantDim(strideAttr, builder);
119  Value divide = builder.create<arith::DivUIOp>(subtract, stride);
120  return builder.create<arith::AddIOp>(divide, one);
121 }
122 
123 // Creates a vector of the dynamic output dims for Conv2D and Depthwise_Conv2D
125  Location loc, Value input, Value weight, ShapedType resultTy,
126  ArrayRef<int64_t> padAttr, ArrayRef<int64_t> strideAttr,
127  ArrayRef<int64_t> dilationAttr, ArrayRef<int64_t> inputSizeDims,
128  ArrayRef<int64_t> kernelSizeDims, OpBuilder &rewriter) {
129  ShapedType inputTy = cast<ShapedType>(input.getType());
130  Type inputETy = inputTy.getElementType();
131  int64_t inputRank = inputTy.getRank();
132 
133  SmallVector<Value> dynDims;
134  dynDims.resize(resultTy.getRank());
135 
136  for (uint32_t i = 0, s = inputSizeDims.size(); i < s; ++i) {
137  int64_t inputDim = inputSizeDims[i];
138  int64_t kernelDim = kernelSizeDims[i];
139  if (inputTy.isDynamicDim(inputDim)) {
140  auto padTop = padAttr[i * 2];
141  auto padBottom = padAttr[i * 2 + 1];
142  auto stride = strideAttr[i];
143  auto dilation = dilationAttr[i];
144  Value initDynDim = rewriter.create<tensor::DimOp>(loc, input, inputDim);
145  Value kernelDynDim =
146  rewriter.create<tensor::DimOp>(loc, weight, kernelDim);
147  // H = F(IH, pad_top, pad_bottom, dilation_y, KH, stride_y)
148  dynDims[inputDim] =
149  getConvOutputDim(loc, initDynDim, padTop, padBottom, kernelDynDim,
150  stride, dilation, inputETy, rewriter);
151  }
152  }
153 
154  // Get the batch/channels dimensions.
155  for (int i = 0; i < inputRank; i++) {
156  if (inputTy.isDynamicDim(i) && !dynDims[i])
157  dynDims[i] = rewriter.create<tensor::DimOp>(loc, input, i);
158  }
159 
160  SmallVector<Value> filteredDims = condenseValues(dynDims);
161  return filteredDims;
162 }
163 
164 // Creates a map to collapse the last dimension of the Depthwise convolution op
165 // due to a shape mismatch
167  int64_t outputRank, SmallVector<ReassociationExprs, 4> &reassociationMap,
168  OpBuilder &rewriter) {
169  reassociationMap.resize(outputRank);
170  for (int i = 0; i < outputRank; i++) {
171  reassociationMap[i].push_back(rewriter.getAffineDimExpr(i));
172  }
173  reassociationMap[outputRank - 1].push_back(
174  rewriter.getAffineDimExpr(outputRank));
175 }
176 
177 namespace {
178 
179 template <typename TosaConvOp, typename LinalgConvOp, typename LinalgConvQOp>
180 class ConvConverter : public OpConversionPattern<TosaConvOp> {
181 public:
184  matchAndRewrite(TosaConvOp op, typename TosaConvOp::Adaptor adaptor,
185  ConversionPatternRewriter &rewriter) const final {
186  Location loc = op->getLoc();
187  Value input = op->getOperand(0);
188  Value weight = op->getOperand(1);
189  Value bias = op->getOperand(2);
190 
191  ShapedType inputTy = cast<ShapedType>(input.getType());
192  ShapedType weightTy = cast<ShapedType>(weight.getType());
193  ShapedType biasTy = cast<ShapedType>(bias.getType());
194  ShapedType resultTy = cast<ShapedType>(op->getResult(0).getType());
195 
196  Type inputETy = inputTy.getElementType();
197  Type resultETy = resultTy.getElementType();
198 
199  DenseI64ArrayAttr padAttr = op.getPadAttr();
200  DenseI64ArrayAttr strideTosaAttr = op.getStrideAttr();
201  DenseI64ArrayAttr dilationTosaAttr = op.getDilationAttr();
202  bool isQuantized = op.getQuantizationInfo().has_value();
203 
204  if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
205  return rewriter.notifyMatchFailure(
206  op, "tosa.conv ops require static shapes for weight and bias");
207 
208  if (inputETy.isUnsignedInteger())
209  return rewriter.notifyMatchFailure(
210  op, "tosa.conv ops does not support unsigned integer input");
211 
212  llvm::SmallVector<int64_t> inputSizeDims;
213  llvm::SmallVector<int64_t> kernelSizeDims;
214  for (int i = 1; i < resultTy.getRank() - 1; i++) {
215  inputSizeDims.push_back(i);
216  kernelSizeDims.push_back(i);
217  }
218 
220  loc, input, weight, resultTy, padAttr.asArrayRef(),
221  strideTosaAttr.asArrayRef(), dilationTosaAttr.asArrayRef(),
222  inputSizeDims, kernelSizeDims, rewriter);
223 
224  auto weightShape = weightTy.getShape();
225 
226  // Apply padding as necessary.
227  TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
228  if (isQuantized) {
229  auto quantizationInfo = *op.getQuantizationInfo();
230  int64_t iZp = quantizationInfo.getInputZp();
231 
232  int64_t intMin =
233  APInt::getSignedMinValue(inputETy.getIntOrFloatBitWidth())
234  .getSExtValue();
235  int64_t intMax =
236  APInt::getSignedMaxValue(inputETy.getIntOrFloatBitWidth())
237  .getSExtValue();
238 
239  if (iZp < intMin || iZp > intMax)
240  return rewriter.notifyMatchFailure(
241  op, "tosa.conv op quantization has zp outside of input range");
242 
243  zeroAttr = rewriter.getIntegerAttr(inputETy, iZp);
244  }
245 
247  pad.resize(2, 0);
248  llvm::append_range(pad, padAttr.asArrayRef());
249  pad.resize(pad.size() + 2, 0);
250  input = applyPad(loc, input, pad, zeroAttr, rewriter);
251 
252  if (4 == inputTy.getRank()) {
253  // For 2D convolutions, we need to check if the target convolution op
254  // wants a HWCF kernel layout.
255  bool wantHwcf =
256  isQuantized ? std::is_same_v<LinalgConvQOp, linalg::Conv2DNhwcHwcfQOp>
257  : std::is_same_v<LinalgConvOp, linalg::Conv2DNhwcHwcfOp>;
258  if (wantHwcf) {
259  // Transpose the kernel to match dimension ordering of the linalg
260  // convolution operation.
261  // TODO(suderman): See if this can be efficiently folded - check whether
262  // the input is used anywhere else, if not fold the constant.
263  SmallVector<int64_t> weightPerm;
264  for (int i = 1; i < resultTy.getRank(); i++)
265  weightPerm.push_back(i);
266  weightPerm.push_back(0);
267 
268  SmallVector<int64_t> newWeightShape;
269  for (auto dim : weightPerm)
270  newWeightShape.push_back(weightShape[dim]);
271  auto weightPermAttr = rewriter.getI64TensorAttr(weightPerm);
272  Value weightPermValue =
273  rewriter.create<arith::ConstantOp>(loc, weightPermAttr);
274  Type newWeightTy =
275  RankedTensorType::get(newWeightShape, weightTy.getElementType());
276  weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
277  weightPermValue);
278  }
279  }
280 
281  // For Conv3D transpose the kernel to match dimension ordering of the linalg
282  // convolution operation. Conv2D has a 1-1 mapping in linalg so better to
283  // map directly and then transpose later if desired.
284  if (5 == inputTy.getRank()) {
285  // TODO(suderman): See if this can be efficiently folded - check whether
286  // the input is used anywhere else, if not fold the constant.
287  SmallVector<int64_t> weightPerm;
288  for (int i = 1; i < resultTy.getRank(); i++)
289  weightPerm.push_back(i);
290  weightPerm.push_back(0);
291 
292  SmallVector<int64_t> newWeightShape;
293  for (auto dim : weightPerm)
294  newWeightShape.push_back(weightShape[dim]);
295  auto weightPermAttr = rewriter.getI64TensorAttr(weightPerm);
296  Value weightPermValue =
297  rewriter.create<arith::ConstantOp>(loc, weightPermAttr);
298  Type newWeightTy =
299  RankedTensorType::get(newWeightShape, weightTy.getElementType());
300  weight = rewriter.create<tosa::TransposeOp>(loc, newWeightTy, weight,
301  weightPermValue);
302  }
303 
304  auto resultZeroAttr = rewriter.getZeroAttr(resultETy);
305  Value emptyTensor = rewriter.create<tensor::EmptyOp>(
306  loc, resultTy.getShape(), resultETy, filteredDims);
307  Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
308  Value zeroTensor = rewriter
309  .create<linalg::FillOp>(loc, ValueRange{zero},
310  ValueRange{emptyTensor})
311  .result();
312 
313  // Extract the attributes for convolution.
314  ArrayRef<int64_t> stride = strideTosaAttr;
315  ArrayRef<int64_t> dilation = dilationTosaAttr;
316 
317  // Create the convolution op.
318  auto strideAttr = rewriter.getI64TensorAttr(stride);
319  auto dilationAttr = rewriter.getI64TensorAttr(dilation);
320 
321  // Create maps for the bias broadcasting
322  SmallVector<AffineMap, 4> indexingMaps;
323  indexingMaps.push_back(AffineMap::get(
324  /*dimCount=*/resultTy.getRank(), /*symbolCount=*/0,
325  {rewriter.getAffineDimExpr(resultTy.getRank() - 1)},
326  rewriter.getContext()));
327  indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultTy.getRank()));
328  indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultTy.getRank()));
329 
330  Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
331  loc, resultTy.getShape(), resultETy, filteredDims);
332 
333  if (isQuantized) {
334  auto quantizationInfo = *op.getQuantizationInfo();
335  auto iZp = rewriter.getI32IntegerAttr(quantizationInfo.getInputZp());
336  auto kZp = rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp());
337 
338  auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
339  auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
340  Value conv =
341  rewriter
342  .create<LinalgConvQOp>(
343  loc, resultTy, ValueRange{input, weight, iZpVal, kZpVal},
344  ValueRange{zeroTensor}, strideAttr, dilationAttr)
345  ->getResult(0);
346  Value result = linalgIntBroadcastExtSIAdd(rewriter, loc, bias, conv,
347  biasEmptyTensor, indexingMaps);
348  rewriter.replaceOp(op, result);
349  return success();
350  }
351 
352  Value conv = rewriter
353  .create<LinalgConvOp>(
354  loc, resultTy, ValueRange{input, weight},
355  ValueRange{zeroTensor}, strideAttr, dilationAttr)
356  ->getResult(0);
357 
358  Value result =
359  rewriter
360  .create<linalg::GenericOp>(
361  loc, resultTy, ValueRange({bias, conv}), biasEmptyTensor,
362  indexingMaps, getNParallelLoopsAttrs(resultTy.getRank()),
363  [&](OpBuilder &nestedBuilder, Location nestedLoc,
364  ValueRange args) {
365  Value added = nestedBuilder.create<arith::AddFOp>(
366  loc, args[0], args[1]);
367  nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
368  })
369  .getResult(0);
370 
371  rewriter.replaceOp(op, result);
372  return success();
373  }
374 };
375 
376 class DepthwiseConvConverter
377  : public OpConversionPattern<tosa::DepthwiseConv2DOp> {
378 public:
381  matchAndRewrite(tosa::DepthwiseConv2DOp op, OpAdaptor adaptor,
382  ConversionPatternRewriter &rewriter) const final {
383  Location loc = op->getLoc();
384  Value input = op->getOperand(0);
385  Value weight = op->getOperand(1);
386  Value bias = op->getOperand(2);
387 
388  ShapedType inputTy = cast<ShapedType>(input.getType());
389  ShapedType weightTy = cast<ShapedType>(weight.getType());
390  ShapedType biasTy = cast<ShapedType>(bias.getType());
391  ShapedType resultTy = cast<ShapedType>(op->getResult(0).getType());
392  int64_t resultRank = resultTy.getRank();
393 
394  Type inputETy = inputTy.getElementType();
395  Type resultETy = resultTy.getElementType();
396 
397  auto padAttr = cast<DenseI64ArrayAttr>(op->getAttr("pad"));
398  auto strideTosaAttr = cast<DenseI64ArrayAttr>(op->getAttr("stride"));
399  auto dilationTosaAttr = cast<DenseI64ArrayAttr>(op->getAttr("dilation"));
400 
401  if (!weightTy.hasStaticShape() || !biasTy.hasStaticShape())
402  return rewriter.notifyMatchFailure(
403  op, "tosa.depthwise_conv ops require static shapes");
404 
405  // Compute output dynamic dims
407  loc, input, weight, resultTy, padAttr.asArrayRef(),
408  strideTosaAttr.asArrayRef(), dilationTosaAttr.asArrayRef(),
409  /*inputSizeDims=*/{1, 2},
410  /*kernelSizeDims=*/{0, 1}, rewriter);
411 
412  bool isQuantized = op->hasAttr("quantization_info");
413  IntegerAttr iZp;
414  IntegerAttr kZp;
415  if (isQuantized) {
416  auto quantizationInfo =
417  cast<tosa::ConvOpQuantizationAttr>(op->getAttr("quantization_info"));
418  iZp = rewriter.getI32IntegerAttr(quantizationInfo.getInputZp());
419  kZp = rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp());
420  }
421 
422  auto weightShape = weightTy.getShape();
423  auto resultShape = resultTy.getShape();
424 
425  // Apply padding as necessary.
426  TypedAttr zeroAttr = rewriter.getZeroAttr(inputETy);
427  if (isQuantized) {
428  auto quantizationInfo =
429  cast<tosa::ConvOpQuantizationAttr>(op->getAttr("quantization_info"));
430  int64_t iZp = quantizationInfo.getInputZp();
431 
432  int64_t intMin =
433  APInt::getSignedMinValue(inputETy.getIntOrFloatBitWidth())
434  .getSExtValue();
435  int64_t intMax =
436  APInt::getSignedMaxValue(inputETy.getIntOrFloatBitWidth())
437  .getSExtValue();
438 
439  if (iZp < intMin || iZp > intMax)
440  return rewriter.notifyMatchFailure(
441  op, "tosa.depthwise_conv op quantization has zp outside of input "
442  "range");
443 
444  zeroAttr = rewriter.getIntegerAttr(inputETy, iZp);
445  }
446 
448  pad.resize(2, 0);
449  llvm::append_range(pad, padAttr.asArrayRef());
450  pad.resize(pad.size() + 2, 0);
451 
452  input = applyPad(loc, input, pad, zeroAttr, rewriter);
453 
454  // Extract the attributes for convolution.
455  ArrayRef<int64_t> stride = strideTosaAttr;
456  ArrayRef<int64_t> dilation = dilationTosaAttr;
457 
458  // Create the convolution op.
459  auto strideAttr = rewriter.getI64TensorAttr(stride);
460  auto dilationAttr = rewriter.getI64TensorAttr(dilation);
461  ShapedType linalgConvTy =
462  RankedTensorType::get({resultShape[0], resultShape[1], resultShape[2],
463  weightShape[2], weightShape[3]},
464  resultETy);
465 
466  // Broadcast the initial value to the output tensor before convolving.
467  SmallVector<AffineMap, 4> indexingMaps;
468  indexingMaps.push_back(AffineMap::get(
469  /*dimCount=*/resultRank, /*symbolCount=*/0,
470  {rewriter.getAffineDimExpr(3)}, rewriter.getContext()));
471  indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
472  indexingMaps.push_back(rewriter.getMultiDimIdentityMap(resultRank));
473 
474  auto resultZeroAttr = rewriter.getZeroAttr(resultETy);
475  Value emptyTensor = rewriter.create<tensor::EmptyOp>(
476  loc, linalgConvTy.getShape(), resultETy, filteredDims);
477  Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
478  Value zeroTensor = rewriter
479  .create<linalg::FillOp>(loc, ValueRange{zero},
480  ValueRange{emptyTensor})
481  .result();
482 
483  Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
484  loc, resultTy.getShape(), resultETy, filteredDims);
485  if (!isQuantized) {
486  Value conv = rewriter
487  .create<linalg::DepthwiseConv2DNhwcHwcmOp>(
488  loc, linalgConvTy, ValueRange{input, weight},
489  ValueRange{zeroTensor}, strideAttr, dilationAttr)
490  .getResult(0);
491 
492  SmallVector<ReassociationExprs, 4> reassociationMap;
493  createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
494  Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
495  loc, resultTy, conv, reassociationMap);
496 
497  Value result =
498  rewriter
499  .create<linalg::GenericOp>(
500  loc, resultTy, ValueRange({bias, convReshape}),
501  biasEmptyTensor, indexingMaps,
502  getNParallelLoopsAttrs(resultRank),
503  [&](OpBuilder &nestedBuilder, Location nestedLoc,
504  ValueRange args) {
505  Value added = nestedBuilder.create<arith::AddFOp>(
506  loc, args[0], args[1]);
507  nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
508  })
509  .getResult(0);
510  rewriter.replaceOp(op, result);
511  } else {
512  auto iZpVal = rewriter.create<arith::ConstantOp>(loc, iZp);
513  auto kZpVal = rewriter.create<arith::ConstantOp>(loc, kZp);
514  Value conv =
515  rewriter
516  .create<linalg::DepthwiseConv2DNhwcHwcmQOp>(
517  loc, linalgConvTy, ValueRange{input, weight, iZpVal, kZpVal},
518  ValueRange{zeroTensor}, strideAttr, dilationAttr)
519  .getResult(0);
520  SmallVector<ReassociationExprs, 4> reassociationMap;
521  createDepthwiseConvCollapseMap(resultRank, reassociationMap, rewriter);
522  Value convReshape = rewriter.create<tensor::CollapseShapeOp>(
523  loc, resultTy, conv, reassociationMap);
525  rewriter, loc, bias, convReshape, biasEmptyTensor, indexingMaps);
526  rewriter.replaceOp(op, result);
527  }
528  return success();
529  }
530 };
531 
532 class MatMulConverter : public OpConversionPattern<tosa::MatMulOp> {
533 public:
536  matchAndRewrite(tosa::MatMulOp op, OpAdaptor adaptor,
537  ConversionPatternRewriter &rewriter) const final {
538  Location loc = op.getLoc();
539 
540  auto outputTy = cast<ShapedType>(op.getType());
541  auto outputElementTy = outputTy.getElementType();
542 
543  SmallVector<Value> dynDims;
544  dynDims.resize(cast<ShapedType>(op->getResult(0).getType()).getRank());
545 
546  if (!outputTy.hasRank() || outputTy.isDynamicDim(0)) {
547  dynDims[0] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 0);
548  }
549 
550  if (!outputTy.hasRank() || outputTy.isDynamicDim(1)) {
551  dynDims[1] = rewriter.create<tensor::DimOp>(loc, op->getOperand(0), 1);
552  }
553 
554  if (!outputTy.hasRank() || outputTy.isDynamicDim(2)) {
555  dynDims[2] = rewriter.create<tensor::DimOp>(loc, op->getOperand(1), 2);
556  }
557 
558  SmallVector<Value> filteredDims = condenseValues(dynDims);
559 
560  auto zeroAttr = rewriter.getZeroAttr(outputElementTy);
561  Value zero = rewriter.create<arith::ConstantOp>(loc, zeroAttr);
562  auto emptyTensor = rewriter.create<tensor::EmptyOp>(
563  loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
564  Value zeroTensor = rewriter
565  .create<linalg::FillOp>(loc, ValueRange{zero},
566  ValueRange{emptyTensor})
567  .result();
568  if (!op.getQuantizationInfo()) {
569  rewriter.replaceOpWithNewOp<linalg::BatchMatmulOp>(
570  op, TypeRange{op.getType()},
571  ValueRange{adaptor.getA(), adaptor.getB()}, ValueRange{zeroTensor});
572  return success();
573  }
574 
575  auto quantizationInfo = *op.getQuantizationInfo();
576  auto aZp = rewriter.create<arith::ConstantOp>(
577  loc, rewriter.getI32IntegerAttr(quantizationInfo.getAZp()));
578  auto bZp = rewriter.create<arith::ConstantOp>(
579  loc, rewriter.getI32IntegerAttr(quantizationInfo.getBZp()));
580  rewriter.replaceOpWithNewOp<linalg::QuantizedBatchMatmulOp>(
581  op, TypeRange{op.getType()},
582  ValueRange{adaptor.getA(), adaptor.getB(), aZp, bZp}, zeroTensor);
583 
584  return success();
585  }
586 };
587 
588 class FullyConnectedConverter
589  : public OpConversionPattern<tosa::FullyConnectedOp> {
590 public:
593  matchAndRewrite(tosa::FullyConnectedOp op, OpAdaptor adaptor,
594  ConversionPatternRewriter &rewriter) const final {
595  Location loc = op.getLoc();
596  auto outputTy = cast<ShapedType>(op.getType());
597  auto input = op.getInput();
598  auto inputTy = cast<ShapedType>(input.getType());
599 
600  auto bias = op.getBias();
601 
602  auto weight = op.getWeight();
603  auto weightTy = cast<ShapedType>(weight.getType());
604  auto weightShape = weightTy.getShape();
605 
606  auto outputETy = outputTy.getElementType();
607 
608  SmallVector<Value> dynDims;
609  dynDims.resize(cast<ShapedType>(op->getResult(0).getType()).getRank());
610 
611  if (!inputTy.hasRank() || inputTy.isDynamicDim(0)) {
612  dynDims[0] = rewriter.create<tensor::DimOp>(loc, input, 0);
613  }
614 
615  if (!weightTy.hasRank() || weightTy.isDynamicDim(0)) {
616  dynDims[1] = rewriter.create<tensor::DimOp>(loc, weight, 0);
617  }
618 
619  SmallVector<Value> filteredDims = condenseValues(dynDims);
620 
621  // Creating maps for the output of MatMul and the bias
622  SmallVector<AffineMap, 4> indexingMaps;
623 
624  // Broadcast the bias.
625  indexingMaps.push_back(AffineMap::get(/*dimCount=*/2, /*symbolCount=*/0,
626  {rewriter.getAffineDimExpr(1)},
627  rewriter.getContext()));
628 
629  indexingMaps.push_back(rewriter.getMultiDimIdentityMap(outputTy.getRank()));
630  indexingMaps.push_back(rewriter.getMultiDimIdentityMap(outputTy.getRank()));
631 
632  auto emptyTensor = rewriter.create<tensor::EmptyOp>(
633  loc, outputTy.getShape(), outputTy.getElementType(), filteredDims);
634 
635  // When quantized, the input elemeny type is not the same as the output
636  auto resultZeroAttr = rewriter.getZeroAttr(outputETy);
637  Value zero = rewriter.create<arith::ConstantOp>(loc, resultZeroAttr);
638  Value zeroTensor = rewriter
639  .create<linalg::FillOp>(loc, ValueRange{zero},
640  ValueRange{emptyTensor})
641  .result();
642 
643  SmallVector<int64_t> permutation{1, 0};
644  auto permutationAttr = rewriter.getI64TensorAttr(permutation);
645  Value permutationValue =
646  rewriter.create<arith::ConstantOp>(loc, permutationAttr);
647 
648  SmallVector<int64_t> newWeightShape{weightShape[1], weightShape[0]};
649  Type newWeightTy =
650  RankedTensorType::get(newWeightShape, weightTy.getElementType());
651 
652  Value transposedWeight = rewriter.create<tosa::TransposeOp>(
653  loc, newWeightTy, weight, permutationValue);
654 
655  Value biasEmptyTensor = rewriter.create<tensor::EmptyOp>(
656  loc, outputTy.getShape(), outputETy, filteredDims);
657 
658  if (!op.getQuantizationInfo()) {
659  Value matmul = rewriter
660  .create<linalg::MatmulOp>(
661  loc, TypeRange{op.getType()},
662  ValueRange{input, transposedWeight}, zeroTensor)
663  ->getResult(0);
664 
665  Value result =
666  rewriter
667  .create<linalg::GenericOp>(
668  loc, outputTy, ValueRange({bias, matmul}), biasEmptyTensor,
669  indexingMaps, getNParallelLoopsAttrs(outputTy.getRank()),
670  [&](OpBuilder &nestedBuilder, Location nestedLoc,
671  ValueRange args) {
672  Value added = nestedBuilder.create<arith::AddFOp>(
673  loc, args[0], args[1]);
674  nestedBuilder.create<linalg::YieldOp>(nestedLoc, added);
675  })
676  .getResult(0);
677  rewriter.replaceOp(op, result);
678  return success();
679  }
680 
681  auto quantizationInfo = *op.getQuantizationInfo();
682  auto inputZp = rewriter.create<arith::ConstantOp>(
683  loc, rewriter.getI32IntegerAttr(quantizationInfo.getInputZp()));
684  auto outputZp = rewriter.create<arith::ConstantOp>(
685  loc, rewriter.getI32IntegerAttr(quantizationInfo.getWeightZp()));
686  Value matmul =
687  rewriter
688  .create<linalg::QuantizedMatmulOp>(
689  loc, TypeRange{op.getType()},
690  ValueRange{input, transposedWeight, inputZp, outputZp},
691  zeroTensor)
692  ->getResult(0);
693  Value result = linalgIntBroadcastExtSIAdd(rewriter, loc, bias, matmul,
694  biasEmptyTensor, indexingMaps);
695  rewriter.replaceOp(op, result);
696  return success();
697  }
698 };
699 
700 class MaxPool2dConverter : public OpRewritePattern<tosa::MaxPool2dOp> {
701 public:
703 
704  LogicalResult matchAndRewrite(tosa::MaxPool2dOp op,
705  PatternRewriter &rewriter) const final {
706  Location loc = op.getLoc();
707  Value input = op.getInput();
708  ShapedType inputTy = cast<ShapedType>(input.getType());
709 
710  ShapedType resultTy = cast<ShapedType>(op.getType());
711  Type resultETy = inputTy.getElementType();
712 
713  auto dynamicDimsOr =
714  checkHasDynamicBatchDims(rewriter, op, {input, op.getOutput()});
715  if (!dynamicDimsOr.has_value())
716  return failure();
717  SmallVector<Value> dynamicDims = *dynamicDimsOr;
718 
719  // Determine what the initial value needs to be for the max pool op.
720  TypedAttr initialAttr;
721  if (resultETy.isF32() || resultETy.isBF16() || resultETy.isF16())
722  initialAttr = rewriter.getFloatAttr(
723  resultETy, APFloat::getLargest(
724  cast<FloatType>(resultETy).getFloatSemantics(), true));
725 
726  if (isa<IntegerType>(resultETy))
727  initialAttr = rewriter.getIntegerAttr(
728  resultETy,
729  APInt::getSignedMinValue(resultETy.getIntOrFloatBitWidth()));
730 
731  if (!initialAttr)
732  return rewriter.notifyMatchFailure(
733  op, "Unsupported initial value for tosa.maxpool_2d op");
734 
735  // Apply padding as necessary.
737  pad.resize(2, 0);
738  llvm::append_range(pad, op.getPad());
739  pad.resize(pad.size() + 2, 0);
740  Value paddedInput = applyPad(loc, input, pad, initialAttr, rewriter);
741 
742  Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
743 
744  ArrayRef<int64_t> kernel = op.getKernel();
745  ArrayRef<int64_t> stride = op.getStride();
746 
747  Attribute strideAttr = rewriter.getI64VectorAttr(stride);
748  Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
749 
750  // Create the linalg op that performs pooling.
751  Value emptyTensor = rewriter.create<tensor::EmptyOp>(
752  loc, resultTy.getShape(), resultTy.getElementType(), dynamicDims);
753 
754  Value filledEmptyTensor =
755  rewriter
756  .create<linalg::FillOp>(loc, ValueRange{initialValue},
757  ValueRange{emptyTensor})
758  .result();
759 
760  Value fakeWindowDims =
761  rewriter.create<tensor::EmptyOp>(loc, kernel, resultETy);
762 
763  rewriter.replaceOpWithNewOp<linalg::PoolingNhwcMaxOp>(
764  op, ArrayRef<Type>{resultTy}, ValueRange{paddedInput, fakeWindowDims},
765  filledEmptyTensor, strideAttr, dilationAttr);
766  return success();
767  }
768 };
769 
770 class AvgPool2dConverter : public OpRewritePattern<tosa::AvgPool2dOp> {
771 public:
773 
774  LogicalResult matchAndRewrite(tosa::AvgPool2dOp op,
775  PatternRewriter &rewriter) const final {
776  Location loc = op.getLoc();
777  Value input = op.getInput();
778  ShapedType inputTy = cast<ShapedType>(input.getType());
779  Type inElementTy = inputTy.getElementType();
780 
781  ShapedType resultTy = cast<ShapedType>(op.getType());
782  Type resultETy = cast<ShapedType>(op.getType()).getElementType();
783 
784  Type accETy = op.getAccType();
785  ShapedType accTy = resultTy.clone(accETy);
786 
787  auto dynamicDimsOr =
788  checkHasDynamicBatchDims(rewriter, op, {input, op.getOutput()});
789  if (!dynamicDimsOr.has_value())
790  return failure();
791  SmallVector<Value> dynamicDims = *dynamicDimsOr;
792 
793  // Apply padding as necessary.
795  pad.resize(2, 0);
796  llvm::append_range(pad, op.getPad());
797  pad.resize(pad.size() + 2, 0);
798  TypedAttr padAttr = rewriter.getZeroAttr(inElementTy);
799  // Unsupported element type
800  if (!padAttr)
801  return failure();
802  Value paddedInput = applyPad(loc, input, pad, padAttr, rewriter);
803 
804  auto initialAttr = rewriter.getZeroAttr(accETy);
805  Value initialValue = rewriter.create<arith::ConstantOp>(loc, initialAttr);
806 
807  ArrayRef<int64_t> kernel = op.getKernel();
808  ArrayRef<int64_t> stride = op.getStride();
809 
810  Attribute strideAttr = rewriter.getI64VectorAttr(stride);
811  Attribute dilationAttr = rewriter.getI64VectorAttr({1, 1});
812 
813  // Create the linalg op that performs pooling.
814  Value poolEmptyTensor = rewriter.create<tensor::EmptyOp>(
815  loc, accTy.getShape(), accETy, dynamicDims);
816 
817  Value filledEmptyTensor =
818  rewriter
819  .create<linalg::FillOp>(loc, ValueRange{initialValue},
820  ValueRange{poolEmptyTensor})
821  .result();
822 
823  Value fakeWindowDims =
824  rewriter.create<tensor::EmptyOp>(loc, kernel, accETy);
825 
826  // Sum across the pooled region.
827  Value poolingOp = rewriter
828  .create<linalg::PoolingNhwcSumOp>(
829  loc, ArrayRef<Type>{accTy},
830  ValueRange{paddedInput, fakeWindowDims},
831  filledEmptyTensor, strideAttr, dilationAttr)
832  .getResult(0);
833 
834  // Normalize the summed value by the number of elements grouped in each
835  // pool.
836  Value iH = rewriter.create<tensor::DimOp>(loc, poolingOp, 1);
837  Value iW = rewriter.create<tensor::DimOp>(loc, poolingOp, 2);
838 
839  auto one = rewriter.create<arith::ConstantIndexOp>(loc, 1);
840  iH = rewriter.create<arith::SubIOp>(loc, iH, one);
841  iW = rewriter.create<arith::SubIOp>(loc, iW, one);
842 
843  Value genericEmptyTensor = rewriter.create<tensor::EmptyOp>(
844  loc, resultTy.getShape(), resultETy, dynamicDims);
845 
846  auto affineMap = rewriter.getMultiDimIdentityMap(resultTy.getRank());
847  auto genericOp = rewriter.create<linalg::GenericOp>(
848  loc, ArrayRef<Type>({resultTy}), ValueRange{poolingOp},
849  ValueRange{genericEmptyTensor},
850  ArrayRef<AffineMap>({affineMap, affineMap}),
851  getNParallelLoopsAttrs(resultTy.getRank()),
852  [&](OpBuilder &b, Location loc, ValueRange args) {
853  auto zero = rewriter.create<arith::ConstantIndexOp>(loc, 0);
854 
855  // Determines what the portion of valid input is covered by the
856  // kernel.
857  auto padFn = [&](Value valid, Value pos, int64_t pad) -> Value {
858  if (pad == 0)
859  return valid;
860 
861  auto padVal = rewriter.create<arith::ConstantIndexOp>(loc, pad);
862  Value dpos = rewriter.create<arith::SubIOp>(loc, pos, padVal);
863 
864  Value cmp = rewriter.create<arith::CmpIOp>(
865  loc, arith::CmpIPredicate::slt, dpos, zero);
866  Value offset =
867  rewriter.create<arith::SelectOp>(loc, cmp, dpos, zero);
868  return rewriter.create<arith::AddIOp>(loc, valid, offset)
869  ->getResult(0);
870  };
871 
872  auto coverageFn = [&](int64_t i, Value isize) -> Value {
873  Value strideVal =
874  rewriter.create<arith::ConstantIndexOp>(loc, stride[i - 1]);
875  Value val =
876  rewriter.create<arith::ConstantIndexOp>(loc, kernel[i - 1]);
877 
878  // Find the position relative to the input tensor's ends.
879  Value left = rewriter.create<linalg::IndexOp>(loc, i);
880  Value right = rewriter.create<arith::SubIOp>(loc, isize, left);
881  left = rewriter.create<arith::MulIOp>(loc, left, strideVal);
882  right = rewriter.create<arith::MulIOp>(loc, right, strideVal);
883 
884  // Determine how much padding was included.
885  val = padFn(val, left, pad[i * 2]);
886  val = padFn(val, right, pad[i * 2 + 1]);
887  Value cmp = rewriter.create<arith::CmpIOp>(
888  loc, arith::CmpIPredicate::slt, val, one);
889  return rewriter.create<arith::SelectOp>(loc, cmp, one, val);
890  };
891 
892  // Compute the indices from either end.
893  Value kH3 = coverageFn(1, iH);
894  Value kW3 = coverageFn(2, iW);
895 
896  // Compute the total number of elements and normalize.
897  auto count = rewriter.create<arith::IndexCastOp>(
898  loc, rewriter.getI32Type(),
899  rewriter.create<arith::MulIOp>(loc, kH3, kW3));
900 
901  // Divide by the number of summed values. For floats this is just
902  // a div however for quantized values input normalization had
903  // to be applied.
904  Value poolVal = args[0];
905  if (isa<FloatType>(accETy)) {
906  auto countF = rewriter.create<arith::SIToFPOp>(loc, accETy, count);
907  poolVal = rewriter.create<arith::DivFOp>(loc, poolVal, countF)
908  ->getResult(0);
909  } else {
910 
911  // If we have quantization information we need to apply an offset
912  // for the input zp value.
913  if (op.getQuantizationInfo()) {
914  auto quantizationInfo = *op.getQuantizationInfo();
915  auto inputZp = rewriter.create<arith::ConstantOp>(
916  loc, b.getIntegerAttr(accETy, quantizationInfo.getInputZp()));
917  Value offset =
918  rewriter.create<arith::MulIOp>(loc, accETy, count, inputZp);
919  poolVal =
920  rewriter.create<arith::SubIOp>(loc, accETy, poolVal, offset);
921  }
922 
923  // Compute: k = 32 - count_leading_zeros(value - 1)
924  Value one32 = rewriter.create<arith::ConstantOp>(
925  loc, rewriter.getI32IntegerAttr(1));
926  Value thirtyTwo32 = rewriter.create<arith::ConstantOp>(
927  loc, rewriter.getI32IntegerAttr(32));
928 
929  Value countSubOne =
930  rewriter.create<arith::SubIOp>(loc, count, one32);
931  Value leadingZeros =
932  rewriter.create<math::CountLeadingZerosOp>(loc, countSubOne);
933  Value k =
934  rewriter.create<arith::SubIOp>(loc, thirtyTwo32, leadingZeros);
935 
936  // Compute: numerator = ((1 << 30) + 1) << k
937  Value k64 =
938  rewriter.create<arith::ExtUIOp>(loc, rewriter.getI64Type(), k);
939  Value thirtyShiftPlusOne = rewriter.create<arith::ConstantOp>(
940  loc, rewriter.getI64IntegerAttr((1 << 30) + 1));
941  Value numerator =
942  rewriter.create<arith::ShLIOp>(loc, thirtyShiftPlusOne, k64);
943 
944  // Compute: scale.multiplier = numerator / value;
945  Value count64 = rewriter.create<arith::ExtUIOp>(
946  loc, rewriter.getI64Type(), count);
947  Value multiplier =
948  rewriter.create<arith::DivUIOp>(loc, numerator, count64);
949  multiplier = rewriter.create<arith::TruncIOp>(
950  loc, rewriter.getI32Type(), multiplier);
951 
952  // Compute: scale.shift = 30 + k
953  Value k8 =
954  rewriter.create<arith::TruncIOp>(loc, rewriter.getI8Type(), k);
955  Value thirty8 = rewriter.create<arith::ConstantOp>(
956  loc, rewriter.getI8IntegerAttr(30));
957  Value shift = rewriter.create<arith::AddIOp>(loc, k8, thirty8);
958 
959  auto scaled =
960  rewriter
961  .create<tosa::ApplyScaleOp>(loc, rewriter.getI32Type(),
962  poolVal, multiplier, shift,
963  rewriter.getBoolAttr(false))
964  .getResult();
965 
966  // If we have quantization information we need to apply output
967  // zeropoint.
968  if (op.getQuantizationInfo()) {
969  auto quantizationInfo = *op.getQuantizationInfo();
970  auto outputZp = rewriter.create<arith::ConstantOp>(
971  loc, b.getIntegerAttr(scaled.getType(),
972  quantizationInfo.getOutputZp()));
973  scaled = rewriter.create<arith::AddIOp>(loc, scaled, outputZp)
974  .getResult();
975  }
976 
977  // Apply Clip.
978  int64_t outBitwidth = resultETy.getIntOrFloatBitWidth();
979 
980  auto min = rewriter.create<arith::ConstantIntOp>(
981  loc, APInt::getSignedMinValue(outBitwidth).getSExtValue(),
982  accETy);
983  auto max = rewriter.create<arith::ConstantIntOp>(
984  loc, APInt::getSignedMaxValue(outBitwidth).getSExtValue(),
985  accETy);
986  auto clamp = clampIntHelper(loc, scaled, min, max, rewriter);
987 
988  poolVal = clamp;
989  // Convert type.
990  if (resultETy != clamp.getType()) {
991  poolVal =
992  rewriter.create<arith::TruncIOp>(loc, resultETy, poolVal);
993  }
994  }
995 
996  rewriter.create<linalg::YieldOp>(loc, poolVal);
997  });
998 
999  rewriter.replaceOp(op, genericOp.getResult(0));
1000  return success();
1001  }
1002 };
1003 
1004 } // namespace
1005 
1007  RewritePatternSet *patterns, const TosaToLinalgNamedOptions &options) {
1008  if (options.preferConv2DKernelLayoutHWCF) {
1009  patterns->add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcHwcfOp,
1010  linalg::Conv2DNhwcHwcfQOp>>(
1011  patterns->getContext());
1012  } else {
1013  patterns->add<ConvConverter<tosa::Conv2DOp, linalg::Conv2DNhwcFhwcOp,
1014  linalg::Conv2DNhwcFhwcQOp>>(
1015  patterns->getContext());
1016  }
1017  patterns->add<
1018  // clang-format off
1019  ConvConverter<tosa::Conv3DOp, linalg::Conv3DNdhwcDhwcfOp, linalg::Conv3DNdhwcDhwcfQOp>,
1020  DepthwiseConvConverter,
1021  MatMulConverter,
1022  MaxPool2dConverter,
1023  AvgPool2dConverter,
1024  FullyConnectedConverter>(patterns->getContext());
1025  // clang-format on
1026 }
static llvm::ManagedStatic< PassManagerOptions > options
static Value clamp(ImplicitLocOpBuilder &builder, Value value, Value lowerBound, Value upperBound)
static Value max(ImplicitLocOpBuilder &builder, Value value, Value bound)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static mlir::Value applyPad(Location loc, Value input, ArrayRef< int64_t > pad, TypedAttr padAttr, OpBuilder &rewriter)
static mlir::Value getConvOutputDim(Location loc, Value inputDim, int64_t padBeforeAttr, int64_t padAfterAttr, Value kernelDim, int64_t strideAttr, int64_t dilationAttr, Type inputETy, OpBuilder &rewriter)
static void createDepthwiseConvCollapseMap(int64_t outputRank, SmallVector< ReassociationExprs, 4 > &reassociationMap, OpBuilder &rewriter)
static mlir::Value linalgIntBroadcastExtSIAdd(PatternRewriter &rewriter, Location loc, Value bias, Value conv, Value result, ArrayRef< AffineMap > indexingMaps)
static mlir::Value reifyConstantDim(int64_t attr, ImplicitLocOpBuilder &builder)
static SmallVector< Value > inferDynamicDimsForConv(Location loc, Value input, Value weight, ShapedType resultTy, ArrayRef< int64_t > padAttr, ArrayRef< int64_t > strideAttr, ArrayRef< int64_t > dilationAttr, ArrayRef< int64_t > inputSizeDims, ArrayRef< int64_t > kernelSizeDims, OpBuilder &rewriter)
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
Attributes are known-constant values of operations.
Definition: Attributes.h:25
IntegerAttr getIndexAttr(int64_t value)
Definition: Builders.cpp:124
IntegerAttr getIntegerAttr(Type type, int64_t value)
Definition: Builders.cpp:238
IntegerAttr getI64IntegerAttr(int64_t value)
Definition: Builders.cpp:128
AffineExpr getAffineDimExpr(unsigned position)
Definition: Builders.cpp:353
IndexType getIndexType()
Definition: Builders.cpp:71
This class implements a pattern rewriter for use with ConversionPatterns.
ImplicitLocOpBuilder maintains a 'current location', allowing use of the create<> method without spec...
OpTy create(Args &&...args)
Create an operation of specific op type at the current insertion point and location.
void createOrFold(llvm::SmallVectorImpl< Value > &results, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:63
This class helps build Operations.
Definition: Builders.h:206
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:446
OpConversionPattern is a wrapper around ConversionPattern that allows for matching and rewriting agai...
Value getOperand(unsigned idx)
Definition: Operation.h:345
Attribute getAttr(StringAttr name)
Return the specified attribute if present, null otherwise.
Definition: Operation.h:512
bool hasAttr(StringAttr name)
Return true if the operation has an attribute with the provided name, false otherwise.
Definition: Operation.h:538
Operation * clone(IRMapping &mapper, CloneOptions options=CloneOptions::all())
Create a deep copy of this operation, remapping any operands that use values outside of the operation...
Definition: Operation.cpp:686
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Definition: Operation.h:402
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
static Operation * create(Location location, OperationName name, TypeRange resultTypes, ValueRange operands, NamedAttrList &&attributes, OpaqueProperties properties, BlockRange successors, unsigned numRegions)
Create a new Operation with the specific fields.
Definition: Operation.cpp:66
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
Definition: PatternMatch.h:727
MLIRContext * getContext() const
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:36
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
bool isF32() const
Definition: Types.cpp:51
bool isUnsignedInteger() const
Return true if this is an unsigned integer type (with the specified width).
Definition: Types.cpp:89
bool isF16() const
Definition: Types.cpp:49
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition: Types.cpp:123
bool isBF16() const
Definition: Types.cpp:48
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:378
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Type getType() const
Return the type of this value.
Definition: Value.h:125
Base class for DenseArrayAttr that is instantiated and specialized for each supported element type be...
std::optional< SmallVector< Value > > checkHasDynamicBatchDims(PatternRewriter &rewriter, Op op, ArrayRef< Value > params)
SmallVector< utils::IteratorType > getNParallelLoopsAttrs(unsigned nParallelLoops)
SmallVector< Value > condenseValues(const SmallVector< Value > &values)
Value clampIntHelper(Location loc, Value arg, Value min, Value max, OpBuilder &rewriter)
void populateTosaToLinalgNamedConversionPatterns(RewritePatternSet *patterns, const TosaToLinalgNamedOptions &options)
Populates conversion passes from TOSA dialect to Linalg named operations.
Include the generated interface declarations.
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:62
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:56
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...
Definition: PatternMatch.h:357