MLIR  20.0.0git
Go to the documentation of this file.
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // Implement Winograd Conv2D algorithm. The implementation is based on the
10 // paper: Fast Algorithms for Convolutional Neural Networks
11 // (https://arxiv.org/abs/1509.09308)
12 //
13 //===----------------------------------------------------------------------===//
14
22 #include "llvm/Support/MathExtras.h"
23
24 namespace mlir {
25 namespace linalg {
26
27 namespace {
28
29 // clang-format off
30 /// Winograd Conv2D uses a minimal 2D filtering algorithm to calculate its
31 /// result. The formula of minimal 2D filtering algorithm F(m x m, r x r),
32 /// m is the output dimension and r is the filter dimension, is
33 ///
34 /// Y = A^T x [ (G x g x G^T) x (B^T x d x B) ] x A
35 ///
36 /// g is filter and d is input data. We need to prepare 6 constant
37 /// transformation matrices, G, G^T, B^T, B, A^T, and A for this formula.
38 ///
39 /// The following tables define these constant transformation matrices for
40 /// F(2 x 2, 3 x 3), F(4 x 4, 3 x 3), and F(2 x 2, 5 x 5)
41 constexpr float G_2x2_3x3[] = {
42  -1, 0, 0,
43  1./2, -1./2, 1./2,
44  1./2, 1./2, 1./2,
45  0, 0, 1
46 };
47
48 constexpr float GT_2x2_3x3[] = {
49  -1, 1./2, 1./2, 0,
50  0, -1./2, 1./2, 0,
51  0, 1./2, 1./2, 1
52 };
53
54 constexpr float BT_2x2_3x3[] = {
55  -1, 0, 1, 0,
56  0, -1, 1, 0,
57  0, 1, 1, 0,
58  0, -1, 0, 1
59 };
60
61 constexpr float B_2x2_3x3[] = {
62  -1, 0, 0, 0,
63  0, -1, 1, -1,
64  1, 1, 1, 0,
65  0, 0, 0, 1
66 };
67
68 constexpr float AT_2x2_3x3[] = {
69  1, 1, 1, 0,
70  0, -1, 1, 1
71 };
72
73 constexpr float A_2x2_3x3[] = {
74  1, 0,
75  1, -1,
76  1, 1,
77  0, 1
78 };
79
80 constexpr float G_4x4_3x3[] = {
81  1, 0, 0,
82  -1./3, 1./3, -1./3,
83  -1./3, -1./3, -1./3,
84  1./12, -1./6, 1./3,
85  1./12, 1./6, 1./3,
86  0, 0, 1
87 };
88
89 constexpr float GT_4x4_3x3[] = {
90  1, -1./3, -1./3, 1./12, 1./12, 0,
91  0, 1./3, -1./3, -1./6, 1./6, 0,
92  0, -1./3, -1./3, 1./3, 1./3, 1
93 };
94
95 constexpr float BT_4x4_3x3[] = {
96  1./4, 0, -5./16, 0, 1./16, 0,
97  0, 1./4, -1./4, -1./16, 1./16, 0,
98  0, -1./4, -1./4, 1./16, 1./16, 0,
99  0, 1./4, -1./8, -1./4, 1./8, 0,
100  0, -1./4, -1./8, 1./4, 1./8, 0,
101  0, 1./4, 0, -5./16, 0, 1./16
102 };
103
104 constexpr float B_4x4_3x3[] = {
105  1./4, 0, 0, 0, 0, 0,
106  0, 1./4, -1./4, 1./4, -1./4, 1./4,
107  -5./16, -1./4, -1./4, -1./8, -1./8, 0,
108  0, -1./16, 1./16, -1./4, 1./4, -5./16,
109  1./16, 1./16, 1./16, 1./8, 1./8, 0,
110  0, 0, 0, 0, 0, 1./16
111 };
112
113 constexpr float AT_4x4_3x3[] = {
114  1./8, 1./4, 1./4, 1./8, 1./8, 0,
115  0, -1./4, 1./4, -1./4, 1./4, 0,
116  0, 1./4, 1./4, 1./2, 1./2, 0,
117  0, -1./4, 1./4, -1, 1, 1./2
118 };
119
120 constexpr float A_4x4_3x3[] = {
121  1./8, 0, 0, 0,
122  1./4, -1./4, 1./4, -1./4,
123  1./4, 1./4, 1./4, 1./4,
124  1./8, -1./4, 1./2, -1,
125  1./8, 1./4, 1./2, 1,
126  0, 0, 0, 1./2
127 };
128
129 constexpr float G_2x2_5x5[] = {
130  1, 0, 0, 0, 0,
131  1./6, -1./6, 1./6, -1./6, 1./6,
132  -1./6, -1./6, -1./6, -1./6, -1./6,
133 -4./15, 2./15, -1./15, 1./30, -1./60,
134  1./60, 1./30, 1./15, 2./15, 4./15,
135  0, 0, 0, 0, 1
136 };
137
138 constexpr float GT_2x2_5x5[] = {
139  1, 1./6, -1./6, -4./15, 1./60, 0,
140  0, -1./6, -1./6, 2./15, 1./30, 0,
141  0, 1./6, -1./6, -1./15, 1./15, 0,
142  0, -1./6, -1./6, 1./30, 2./15, 0,
143  0, 1./6, -1./6, -1./60, 4./15, 1
144 };
145
146 constexpr float BT_2x2_5x5[] = {
147  1./8, 3./16, -1./4, -3./16, 1./8, 0,
148  0, 1./8, 1./16, -5./16, 1./8, 0,
149  0, -1./8, -5./16, -1./16, 1./8, 0,
150  0, 1./4, -1./8, -1./4, 1./8, 0,
151  0, -1./8, -1./4, 1./8, 1./4, 0,
152  0, 1./8, 3./16, -1./4, -3./16, 1./8
153 };
154
155 constexpr float B_2x2_5x5[] = {
156  1./8, 0, 0, 0, 0, 0,
157  3./16, 1./8, -1./8, 1./4, -1./8, 1./8,
158  -1./4, 1./16, -5./16, -1./8, -1./4, 3./16,
159  -3./16, -5./16, -1./16, -1./4, 1./8, -1./4,
160  1./8, 1./8, 1./8, 1./8, 1./4, -3./16,
161  0, 0, 0, 0, 0, 1./8
162 };
163
164 constexpr float AT_2x2_5x5[] = {
165  1./2, 1, 1, 2, 1, 0,
166  0, -1, 1, -1, 2, 1./2
167 };
168
169 constexpr float A_2x2_5x5[] = {
170  1./2, 0,
171  1, -1,
172  1, 1,
173  2, -1,
174  1, 2,
175  0, 1./2
176 };
177 // clang-format on
178
179 using TransformMapKeyTy = std::pair<int, int>;
180
181 /// We use F(m, r) to define the size of minimal filtering algorithms.
182 /// m is the output dimension and r is the filter dimension. We can get
183 /// the input dimension, alpha, from the formula, alpha = m + r - 1.
184 ///
185 /// For example, when m = 2 and r = 3, we know its input size is 4.
186 /// The Conv2D will operate on 4x4 input data with 3x3 filter and get
187 /// 2x2 output result.
188 constexpr TransformMapKeyTy F_2_3{2, 3};
189 constexpr TransformMapKeyTy F_4_3{4, 3};
190 constexpr TransformMapKeyTy F_2_5{2, 5};
191
192 /// Structure to keep information of constant transform matrices.
193 struct TransformMatrix {
194  TransformMatrix(const float *table, int64_t rows, int64_t cols,
195  int64_t scalarFactor = 1)
197
198  const float *table;
199  int64_t rows;
200  int64_t cols;
201  int64_t scalarFactor;
202 };
203
204 /// Utility function to convert constant array to arith.constant Value.
205 Value create2DTransformMatrix(OpBuilder &builder, Location loc,
206  TransformMatrix transform, Type type) {
207  ArrayRef<float> constVec(transform.table, transform.rows * transform.cols);
208
209  return builder.create<arith::ConstantOp>(
212  SmallVector<int64_t>{transform.rows, transform.cols}, type),
213  constVec));
214 }
215
216 /// Extract height x width data from 4D tensors.
217 Value extract2DDataFrom4D(OpBuilder &builder, Location loc, Value source,
218  Value loopNorFIndex, Value loopCorFIndex,
219  Value heightOffset, Value widthOffset,
220  int64_t extractHeight, int64_t extractWidth,
221  int64_t loopNorFIdx, int64_t loopCorFIdx,
222  int64_t heightIdx, int64_t widthIdx) {
223  auto sourceType = cast<ShapedType>(source.getType());
224  Type elementType = sourceType.getElementType();
225  int64_t srcSize = sourceType.getRank();
226
227  auto oneIndex = builder.getIndexAttr(1);
228  SmallVector<OpFoldResult> offsets;
229  offsets.resize(srcSize);
230  offsets[loopNorFIdx] = loopNorFIndex;
231  offsets[loopCorFIdx] = loopCorFIndex;
232  offsets[heightIdx] = heightOffset;
233  offsets[widthIdx] = widthOffset;
234  SmallVector<OpFoldResult> sizes(srcSize, oneIndex);
235  sizes[heightIdx] = builder.getIndexAttr(extractHeight);
236  sizes[widthIdx] = builder.getIndexAttr(extractWidth);
237  SmallVector<OpFoldResult> strides(srcSize, oneIndex);
238
239  auto extractFilterType =
240  RankedTensorType::get({extractHeight, extractWidth}, elementType);
241  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
242  loc, extractFilterType, source, offsets, sizes, strides);
243
244  return extractFilterOp;
245 }
246
247 /// Extract height x width data from 6D tensors.
248 Value extract2DDataFrom6D(OpBuilder &builder, Location loc, Value source,
249  Value tileHIndex, Value tileWIndex,
250  Value loopNorFIndex, Value loopCorFIndex,
251  int64_t tileHIdx, int64_t tileWIdx,
252  int64_t loopNorFIdx, int64_t loopCorFIdx,
253  int64_t heightIdx, int64_t widthIdx) {
254  auto sourceType = cast<ShapedType>(source.getType());
255  Type elementType = sourceType.getElementType();
256  auto sourceShape = sourceType.getShape();
257  int64_t srcSize = sourceType.getRank();
258  int64_t height = sourceShape[heightIdx];
259  int64_t width = sourceShape[widthIdx];
260
261  auto zeroIndex = builder.getIndexAttr(0);
262  auto oneIndex = builder.getIndexAttr(1);
263  SmallVector<OpFoldResult> offsets(srcSize, zeroIndex);
264  offsets.resize(srcSize);
265  offsets[tileHIdx] = tileHIndex;
266  offsets[tileWIdx] = tileWIndex;
267  offsets[loopNorFIdx] = loopNorFIndex;
268  offsets[loopCorFIdx] = loopCorFIndex;
269  SmallVector<OpFoldResult> sizes(srcSize, oneIndex);
270  sizes[heightIdx] = builder.getIndexAttr(height);
271  sizes[widthIdx] = builder.getIndexAttr(width);
272  SmallVector<OpFoldResult> strides(srcSize, oneIndex);
273
274  auto extractFilterType = RankedTensorType::get({height, width}, elementType);
275  auto extractFilterOp = builder.create<tensor::ExtractSliceOp>(
276  loc, extractFilterType, source, offsets, sizes, strides);
277
278  return extractFilterOp;
279 }
280
281 /// Insert transformed height x width data to 4D tensors which it is
282 /// extracted from.
283 Value insert2DDataTo4D(OpBuilder &builder, Location loc, Value source,
284  Value dest, Value loopNorFIndex, Value loopCorFIndex,
285  Value heightOffset, Value widthOffset, int64_t height,
286  int64_t width, int64_t loopNorFIdx, int64_t loopCorFIdx,
287  int64_t heightIdx, int64_t widthIdx) {
288  int64_t destSize = cast<ShapedType>(dest.getType()).getRank();
289  auto oneIndex = builder.getIndexAttr(1);
290  SmallVector<OpFoldResult> retOffsets;
291  retOffsets.resize(destSize);
292  retOffsets[loopNorFIdx] = loopNorFIndex;
293  retOffsets[loopCorFIdx] = loopCorFIndex;
294  retOffsets[heightIdx] = heightOffset;
295  retOffsets[widthIdx] = widthOffset;
296  SmallVector<OpFoldResult> retSizes(destSize, oneIndex);
297  retSizes[heightIdx] = builder.getIndexAttr(height);
298  retSizes[widthIdx] = builder.getIndexAttr(width);
299  SmallVector<OpFoldResult> strides(destSize, oneIndex);
300
301  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
302  loc, source, dest, retOffsets, retSizes, strides);
303
304  return insertSliceOp;
305 }
306
307 /// Insert transformed height x width data to 6D tensors which it is
308 /// extracted from.
309 Value insert2DDataTo6D(OpBuilder &builder, Location loc, Value source,
310  Value dest, Value tileHIndex, Value tileWIndex,
311  Value loopNorFIndex, Value loopCorFIndex, int64_t height,
312  int64_t width, int64_t tileHIdx, int64_t tileWIdx,
313  int64_t loopNorFIdx, int64_t loopCorFIdx,
314  int64_t heightIdx, int64_t widthIdx) {
315  int64_t destSize = cast<ShapedType>(dest.getType()).getRank();
316  auto zeroIndex = builder.getIndexAttr(0);
317  auto oneIndex = builder.getIndexAttr(1);
318  SmallVector<OpFoldResult> retOffsets(destSize, zeroIndex);
319  retOffsets.resize(destSize);
320  retOffsets[tileHIdx] = tileHIndex;
321  retOffsets[tileWIdx] = tileWIndex;
322  retOffsets[loopNorFIdx] = loopNorFIndex;
323  retOffsets[loopCorFIdx] = loopCorFIndex;
324  SmallVector<OpFoldResult> retSizes(destSize, oneIndex);
325  retSizes[heightIdx] = builder.getIndexAttr(height);
326  retSizes[widthIdx] = builder.getIndexAttr(width);
327  SmallVector<OpFoldResult> strides(destSize, oneIndex);
328
329  auto insertSliceOp = builder.create<tensor::InsertSliceOp>(
330  loc, source, dest, retOffsets, retSizes, strides);
331
332  return insertSliceOp;
333 }
334
335 /// This function transforms the filter. The data layout of the filter is FHWC.
336 /// The transformation matrix is 2-dimension. We need to extract H x W from
337 /// FHWC first. We need to generate 2 levels of loops to iterate on F and C.
338 /// After the transformation, we get
339 ///
340 /// scf.for %f = lo_f to hi_f step 1
341 /// scf.for %c = lo_c to hi_c step 1
342 /// %extracted = extract filter<h x w> from filter<f x h x w x c>
343 /// %ret = linalg.matmul G, %extracted
344 /// %ret = linalg.matmul %ret, GT
345 /// %inserted = insert %ret into filter<h x w x c x f>
346 Value filterTransform(RewriterBase &rewriter, Location loc, Value filter,
347  Value retValue, int64_t m, int64_t r,
348  bool leftTransform = true, bool rightTransform = true) {
349  // Map from (m, r) to G transform matrix.
350  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
351  GMatrices = {
352  {F_2_3, TransformMatrix(G_2x2_3x3, 4, 3)},
353  {F_4_3, TransformMatrix(G_4x4_3x3, 6, 3)},
354  {F_2_5, TransformMatrix(G_2x2_5x5, 6, 5)},
355  };
356
357  // Map from (m, r) to GT transform matrix.
358  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
359  GTMatrices = {
360  {F_2_3, TransformMatrix(GT_2x2_3x3, 3, 4)},
361  {F_4_3, TransformMatrix(GT_4x4_3x3, 3, 6)},
362  {F_2_5, TransformMatrix(GT_2x2_5x5, 5, 6)},
363  };
364
365  auto filterType = cast<ShapedType>(filter.getType());
366  Type elementType = filterType.getElementType();
367  auto filterShape = filterType.getShape(); // F, H, W, C
368  int64_t filterF = filterShape[0];
369  int64_t filterH = filterShape[1];
370  int64_t filterW = filterShape[2];
371  int64_t filterC = filterShape[3];
372
373  if (filterH != r && filterH != 1)
374  return Value();
375  if (filterW != r && filterW != 1)
376  return Value();
377
378  Value zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
379  auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
380  ValueRange args) -> scf::ValueVector {
381  Value FIter = ivs[0];
382  Value CIter = ivs[1];
383
384  // Extract (H, W) from (F, H, W, C).
385  auto extractFilter =
386  extract2DDataFrom4D(builder, loc, filter, FIter, CIter, zeroIdx,
387  zeroIdx, filterH, filterW, /*loopNorFIdx=*/0,
388  /*loopCorFIdx=*/3, /*heightIdx=*/1, /*widthIdx=*/2);
389
390  TransformMapKeyTy key = {m, r};
391  int64_t retRows = 1;
392  Value matmulRetValue = extractFilter;
393  Value zero = builder.create<arith::ConstantOp>(
394  loc, rewriter.getZeroAttr(elementType));
395  if (leftTransform) {
396  // Get constant transform matrix G.
397  auto it = GMatrices.find(key);
398  if (it == GMatrices.end())
399  return {};
400  const TransformMatrix &GMatrix = it->second;
401
402  retRows = GMatrix.rows;
403  auto matmulType = RankedTensorType::get({retRows, filterW}, elementType);
404  auto empty =
405  builder
406  .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
407  .getResult();
408  auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
409
410  Value G = create2DTransformMatrix(builder, loc, GMatrix, elementType);
411  // Multiply G x g.
412  auto matmulOp = builder.create<linalg::MatmulOp>(
413  loc, matmulType, ValueRange{G, extractFilter}, ValueRange{init});
414  matmulRetValue = matmulOp.getResult(0);
415  }
416
417  if (rightTransform) {
418  // Get constant transform matrix GT.
419  auto it = GTMatrices.find(key);
420  if (it == GTMatrices.end())
421  return {};
422  const TransformMatrix &GTMatrix = it->second;
423
424  auto matmulType =
425  RankedTensorType::get({retRows, GTMatrix.cols}, elementType);
426  auto empty =
427  builder
428  .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
429  .getResult();
430  auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
431
432  Value GT = create2DTransformMatrix(builder, loc, GTMatrix, elementType);
433  // Multiply u = (G x g) x GT.
434  auto matmulOp = builder.create<linalg::MatmulOp>(
435  loc, matmulType, ValueRange{matmulRetValue, GT}, ValueRange{init});
436  matmulRetValue = matmulOp.getResult(0);
437  }
438
439  // Insert (H, W) to (H, W, C, F).
440  int64_t retHeight = leftTransform ? m + r - 1 : 1;
441  int64_t retWidth = rightTransform ? m + r - 1 : 1;
442
443  auto insertSliceOp =
444  insert2DDataTo4D(builder, loc, matmulRetValue, args[0], FIter, CIter,
445  zeroIdx, zeroIdx, retHeight, retWidth,
446  /*loopNorFIdx=*/3, /*loopCorFIdx=*/2,
447  /*heightIdx=*/0, /*widthIdx=*/1);
448
449  return {insertSliceOp};
450  };
451
452  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterF);
453  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, filterC);
454  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
455  scf::LoopNest loops = scf::buildLoopNest(
456  rewriter, loc, {zeroIdx, zeroIdx}, {fUpperBound, cUpperBound},
457  {oneStep, oneStep}, {retValue}, buildBody);
458  return loops.results[0];
459 }
460
461 /// This function transforms the input. The data layout of the input is NHWC.
462 /// The transformation matrix is 2-dimension. We need to extract H x W from
463 /// NHWC first. We need to generate 2 levels of loops to iterate on N and C.
464 /// After the transformation, we get
465 ///
466 /// scf.for %h = 0 to tileH step 1
467 /// scf.for %w = 0 to tileW step 1
468 /// scf.for %n = 0 to N step 1
469 /// scf.for %c = 0 to C step 1
470 /// %extracted = extract %extracted<alphaH x alphaW> from
471 /// %input<N x H x W x C>
472 /// at [%n, (%h x m), (%w x m), %c]
473 /// %ret = linalg.matmul BT, %extracted
474 /// %ret = linalg.matmul %ret, B
475 /// %inserted = insert %ret<alphaH x alphaW> into
476 /// %output<alphaH x alphaW x tileH x tileW x N x C>
477 /// at [0, 0, %h, %w, %n, %c]
478 Value inputTransform(RewriterBase &rewriter, Location loc, Value input,
479  Value retValue, int64_t m, int64_t r,
480  bool leftTransform = true, bool rightTransform = true) {
481  // Map from (m, r) to BT transform matrix.
482  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
483  BTMatrices = {
484  {F_2_3, TransformMatrix(BT_2x2_3x3, 4, 4)},
485  {F_4_3, TransformMatrix(BT_4x4_3x3, 6, 6)},
486  {F_2_5, TransformMatrix(BT_2x2_5x5, 6, 6)},
487  };
488
489  // Map from (m, r) to B transform matrix.
490  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
491  BMatrices = {
492  {F_2_3, TransformMatrix(B_2x2_3x3, 4, 4)},
493  {F_4_3, TransformMatrix(B_4x4_3x3, 6, 6)},
494  {F_2_5, TransformMatrix(B_2x2_5x5, 6, 6)},
495  };
496
497  auto inputType = cast<ShapedType>(input.getType());
498  Type elementType = inputType.getElementType();
499  auto inputShape = inputType.getShape(); // N, H, W, C
500  int64_t inputN = inputShape[0];
501  int64_t inputC = inputShape[3];
502  auto valueType = cast<ShapedType>(retValue.getType());
503  auto valueShape = valueType.getShape(); // alphaH, alphaW, HTile, WTile, N, C
504  int64_t tileH = valueShape[2];
505  int64_t tileW = valueShape[3];
506  int64_t alphaH = leftTransform ? m + r - 1 : 1;
507  int64_t alphaW = rightTransform ? m + r - 1 : 1;
508
509  auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
510  ValueRange args) -> scf::ValueVector {
511  Value tileHIter = ivs[0];
512  Value tileWIter = ivs[1];
513  Value NIter = ivs[2];
514  Value CIter = ivs[3];
515
516  auto context = builder.getContext();
517  auto affineMap =
518  AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
519  Value heightOffset =
520  builder.create<affine::AffineApplyOp>(loc, affineMap, tileHIter);
521  Value widthOffset =
522  builder.create<affine::AffineApplyOp>(loc, affineMap, tileWIter);
523
524  // Extract (H, W) from (N, H, W, C).
525  auto extractInput =
526  extract2DDataFrom4D(builder, loc, input, NIter, CIter, heightOffset,
527  widthOffset, alphaH, alphaW, /*loopNorFIdx=*/0,
528  /*loopCorFIdx=*/3, /*heightIdx=*/1, /*widthIdx=*/2);
529
530  TransformMapKeyTy key = {m, r};
531  int64_t retRows = 1;
532  int64_t retCols = 1;
533  Value matmulRetValue = extractInput;
534  Value zero = builder.create<arith::ConstantOp>(
535  loc, rewriter.getZeroAttr(elementType));
536  if (leftTransform) {
537  // Get constant transform matrix BT.
538  auto it = BTMatrices.find(key);
539  if (it == BTMatrices.end())
540  return {};
541  const TransformMatrix &BTMatrix = it->second;
542
543  retRows = BTMatrix.rows;
544  auto matmulType = RankedTensorType::get({retRows, alphaW}, elementType);
545  auto empty =
546  builder
547  .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
548  .getResult();
549  auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
550
551  Value BT =
552  create2DTransformMatrix(builder, loc, BTMatrix, builder.getF32Type());
553  // Multiply BT x d.
554  auto matmulOp = builder.create<linalg::MatmulOp>(
555  loc, matmulType, ValueRange{BT, matmulRetValue}, ValueRange{init});
556  matmulRetValue = matmulOp.getResult(0);
557  }
558
559  if (rightTransform) {
560  // Get constant transform matrix B.
561  auto it = BMatrices.find(key);
562  if (it == BMatrices.end())
563  return {};
564  const TransformMatrix &BMatrix = it->second;
565
566  retCols = BMatrix.cols;
567  auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
568  auto empty =
569  builder
570  .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
571  .getResult();
572  auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
573  Value B =
574  create2DTransformMatrix(builder, loc, BMatrix, builder.getF32Type());
575  // Multiply v = (BT x d) x B.
576  auto matmulOp = builder.create<linalg::MatmulOp>(
577  loc, matmulType, ValueRange{matmulRetValue, B}, ValueRange{init});
578  matmulRetValue = matmulOp.getResult(0);
579  }
580
581  // Insert (H, W) to (H, W, tileH, tileW, N, C).
582  auto combinedVal = insert2DDataTo6D(
583  builder, loc, matmulRetValue, args[0], tileHIter, tileWIter, NIter,
584  CIter, retRows, retCols, 2, 3, /*loopNorFIdx=*/4, /*loopCorFIdx=*/5,
585  /*heightIdx=*/0, /*widthIdx=*/1);
586
587  return {combinedVal};
588  };
589
590  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
591  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tileH);
592  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
593  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputN);
594  auto cUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, inputC);
595  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
596  scf::LoopNest loops = scf::buildLoopNest(
597  rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
598  {tileHBound, tileWBound, nUpperBound, cUpperBound},
599  {oneStep, oneStep, oneStep, oneStep}, {retValue}, buildBody);
600  return loops.results[0];
601 }
602
603 /// This function generates linalg.batch_matmul to multiply input with filter.
604 /// linalg.batch_matmul only supports 3-dimensional inputs. We can treat
605 /// tileH x tileW x H x W data as the 1-dimensional data array. That is to
606 /// convert [tileH, tileW, H, W, N, C] to [tileH x tileW x H x W, N, C]. In this
607 /// way, we can convert 6-dimensional inputs to 3-dimensional representation
608 /// that is suitable for linalg.batch_matmul.
609 ///
610 /// Batched matmul will do the matrix multiply with the reduction on channel.
611 ///
612 /// We get
613 ///
614 /// %collapsed_input = tensor.collapse_shape %input
615 /// %collapsed_filter = tensor.collapse_shape %filter
616 /// %ret = linalg.batch_matmul %collapsed_input, %collapsed_filter
617 /// %expanded_ret = tensor.expand_shape %ret
618 ///
619 /// After this function, we get return value with data layout
620 /// (tileH, tileW, H, W, N, F).
621 static Value matrixMultiply(RewriterBase &rewriter, Location loc,
622  Value transformedFilter, Value transformedInput,
623  Type outputElementType) {
624  // Convert (alphaH, alphaW, C, F) to (alphaH x alphaW, C, F) for filter.
625  auto filterType = cast<ShapedType>(transformedFilter.getType());
626  assert(filterType.hasStaticShape() && "only support static shapes.");
627  ArrayRef<int64_t> filterShape = filterType.getShape();
628  Type filterElementType = filterType.getElementType();
629  auto filterReassocType = RankedTensorType::get(
630  {filterShape[0] * filterShape[1], filterShape[2], filterShape[3]},
631  filterElementType);
632  SmallVector<ReassociationIndices> filterReassoc = {{0, 1}, {2}, {3}};
633  Value collapseFilter = rewriter.create<tensor::CollapseShapeOp>(
634  loc, filterReassocType, transformedFilter, filterReassoc);
635
636  // Convert (alphaH, alphaW, tileH, tileW, N, C) to
637  // (alphaH x alphaW, tileH x tileW x N, C) for input.
638  auto inputType = cast<ShapedType>(transformedInput.getType());
639  assert(inputType.hasStaticShape() && "only support static shapes.");
640  ArrayRef<int64_t> inputShape = inputType.getShape();
641  Type inputElementType = inputType.getElementType();
642  auto inputReassocType = RankedTensorType::get(
643  {inputShape[0] * inputShape[1],
644  inputShape[2] * inputShape[3] * inputShape[4], inputShape[5]},
645  inputElementType);
646  SmallVector<ReassociationIndices> inputReassoc = {{0, 1}, {2, 3, 4}, {5}};
647  Value collapseInput = rewriter.create<tensor::CollapseShapeOp>(
648  loc, inputReassocType, transformedInput, inputReassoc);
649
650  // Batched matrix multiply.
651  auto matmulType = RankedTensorType::get(
652  {inputShape[0] * inputShape[1],
653  inputShape[2] * inputShape[3] * inputShape[4], filterShape[3]},
654  outputElementType);
655  Value empty = rewriter
656  .create<tensor::EmptyOp>(loc, matmulType.getShape(),
657  outputElementType)
658  .getResult();
659  Value zero = rewriter.create<arith::ConstantOp>(
660  loc, rewriter.getZeroAttr(outputElementType));
661  Value init = rewriter.create<linalg::FillOp>(loc, zero, empty).getResult(0);
662
663  auto matmulOp = rewriter.create<linalg::BatchMatmulOp>(
664  loc, matmulType, ValueRange({collapseInput, collapseFilter}),
665  ValueRange{init});
666
667  // The result shape of batch matmul is (alphaH x alphaW, tileH x tileW x N, F)
668  // Expand matmul result to (alphaH, alphaW, tileH, tileW, N, F).
669  SmallVector<ReassociationIndices> outputReassoc = {{0, 1}, {2, 3, 4}, {5}};
670  auto outputReassocType =
671  RankedTensorType::get({inputShape[0], inputShape[1], inputShape[2],
672  inputShape[3], inputShape[4], filterShape[3]},
673  outputElementType);
674  auto expandOutput = rewriter.create<tensor::ExpandShapeOp>(
675  loc, outputReassocType, matmulOp.getResult(0), outputReassoc);
676  return expandOutput;
677 }
678
679 /// This function transforms the output. The data layout of the output is HWNF.
680 /// The transformation matrix is 2-dimension. We need to extract H x W from
681 /// HWNF first. We need to generate 2 levels of loops to iterate on N and F.
682 /// After the transformation, we get
683 ///
684 /// scf.for %h = 0 to tileH step 1
685 /// scf.for %w = 0 to tileW step 1
686 /// scf.for %n = 0 to N step 1
687 /// scf.for %f = 0 to F step 1
688 /// %extracted = extract %extracted<alphaH x alphaW> from
689 /// %input<alphaH x alphaW x tileH x tileW x N x F>
690 /// at [0, 0, %h, %w, %n, %f]
691 /// %ret = linalg.matmul AT, %extracted
692 /// %ret = linalg.matmul %ret, A
693 /// %inserted = insert %ret<alphaH x alphaW> into
694 /// output<N x H x W x F>
695 /// at [%n, (%h x m), (%w x m), %f]
696 Value outputTransform(RewriterBase &rewriter, Location loc, Value value,
697  Value output, int64_t m, int64_t r,
698  bool leftTransform = true, bool rightTransform = true) {
699  // Map from (m, r) to AT transform matrix.
700  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
701  ATMatrices = {
702  {F_2_3, TransformMatrix(AT_2x2_3x3, 2, 4)},
703  {F_4_3, TransformMatrix(AT_4x4_3x3, 4, 6, 32)},
704  {F_2_5, TransformMatrix(AT_2x2_5x5, 2, 6, 16)},
705  };
706
707  // Map from (m, r) to A transform matrix.
708  static const llvm::SmallDenseMap<TransformMapKeyTy, TransformMatrix>
709  AMatrices = {
710  {F_2_3, TransformMatrix(A_2x2_3x3, 4, 2)},
711  {F_4_3, TransformMatrix(A_4x4_3x3, 6, 4, 32)},
712  {F_2_5, TransformMatrix(A_2x2_5x5, 6, 2, 16)},
713  };
714
715  auto valueType = cast<ShapedType>(value.getType());
716  Type elementType = valueType.getElementType();
717  auto valueShape = valueType.getShape(); // H, W, TileH, TileW, N, F
718  int64_t valueH = valueShape[0];
719  int64_t valueW = valueShape[1];
720  int64_t valueN = valueShape[4];
721  int64_t valueF = valueShape[5];
722  int64_t alphaH = leftTransform ? m + r - 1 : 1;
723  int64_t alphaW = rightTransform ? m + r - 1 : 1;
724
725  if (valueH != alphaH && valueH != 1)
726  return Value();
727  if (valueW != alphaW && valueW != 1)
728  return Value();
729
730  auto buildBody = [&](OpBuilder &builder, Location loc, ValueRange ivs,
731  ValueRange args) -> scf::ValueVector {
732  Value tileHIter = ivs[0];
733  Value tileWIter = ivs[1];
734  Value NIter = ivs[2];
735  Value FIter = ivs[3];
736
737  // Extract (H, W) from (H, W, tileH, tileW, N, F).
738  auto extractValue =
739  extract2DDataFrom6D(builder, loc, value, tileHIter, tileWIter, NIter,
740  FIter, 2, 3, /*loopNorFIdx=*/4,
741  /*loopCorFIdx=*/5, /*heightIdx=*/0, /*widthIdx=*/1);
742
743  TransformMapKeyTy key = {m, r};
744  int64_t retRows = 1;
745  int64_t retCols = 1;
746  int64_t leftScalarFactor = 1;
747  int64_t rightScalarFactor = 1;
748  Value matmulRetValue = extractValue;
749  Value zero = builder.create<arith::ConstantOp>(
750  loc, rewriter.getZeroAttr(elementType));
751  if (leftTransform) {
752  // Get constant transform matrix AT.
753  auto it = ATMatrices.find(key);
754  if (it == ATMatrices.end())
755  return {};
756  const TransformMatrix &ATMatrix = it->second;
757
758  leftScalarFactor = ATMatrix.scalarFactor;
759  retRows = ATMatrix.rows;
760  auto matmulType = RankedTensorType::get({retRows, valueW}, elementType);
761  auto empty =
762  builder
763  .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
764  .getResult();
765  auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
766
767  Value AT = create2DTransformMatrix(builder, loc, ATMatrix, elementType);
768  // Multiply AT x m.
769  auto matmulOp = builder.create<linalg::MatmulOp>(
770  loc, matmulType, ValueRange{AT, matmulRetValue}, ValueRange{init});
771  matmulRetValue = matmulOp.getResult(0);
772  }
773
774  if (rightTransform) {
775  // Get constant transform matrix T.
776  auto it = AMatrices.find(key);
777  if (it == AMatrices.end())
778  return {};
779  const TransformMatrix &AMatrix = it->second;
780
781  rightScalarFactor = AMatrix.scalarFactor;
782  auto matmulType =
783  RankedTensorType::get({retRows, AMatrix.cols}, elementType);
784  retCols = AMatrix.cols;
785  auto empty =
786  builder
787  .create<tensor::EmptyOp>(loc, matmulType.getShape(), elementType)
788  .getResult();
789  auto init = builder.create<linalg::FillOp>(loc, zero, empty).getResult(0);
790
791  Value A = create2DTransformMatrix(builder, loc, AMatrix, elementType);
792  // Multiply y = (AT x m) x A.
793  auto matmulOp = builder.create<linalg::MatmulOp>(
794  loc, matmulType, ValueRange{matmulRetValue, A}, ValueRange{init});
795  matmulRetValue = matmulOp.getResult(0);
796  }
797
798  if (leftScalarFactor * rightScalarFactor != 1) {
799  // Multiply scalar factor.
800  Value scalarFactor = builder.create<arith::ConstantOp>(
801  loc,
802  FloatAttr::get(elementType, leftScalarFactor * rightScalarFactor));
803  auto matmulType = RankedTensorType::get({retRows, retCols}, elementType);
804  auto init = builder.create<tensor::EmptyOp>(loc, matmulType.getShape(),
805  elementType);
806
807  auto identityAffineMap = rewriter.getMultiDimIdentityMap(2);
808  SmallVector<AffineMap> affineMaps = {
809  AffineMap::get(2, 0, init.getContext()), identityAffineMap};
811  rewriter
812  .create<linalg::GenericOp>(
813  loc, matmulType, ValueRange{scalarFactor}, ValueRange{init},
814  affineMaps,
816  utils::IteratorType::parallel,
817  utils::IteratorType::parallel},
818  [&](OpBuilder &nestedBuilder, Location nestedLoc,
819  ValueRange args) {
820  nestedBuilder.create<linalg::YieldOp>(nestedLoc, args[0]);
821  })
822  .getResult(0);
823
824  matmulRetValue = builder
825  .create<linalg::MulOp>(
826  loc, matmulType,
828  ValueRange{init})
829  .getResult(0);
830  }
831
832  auto context = builder.getContext();
833  auto affineMap =
834  AffineMap::get(1, 0, {builder.getAffineDimExpr(0) * m}, context);
835  Value heightOffset =
836  builder.create<affine::AffineApplyOp>(loc, affineMap, tileHIter);
837  Value widthOffset =
838  builder.create<affine::AffineApplyOp>(loc, affineMap, tileWIter);
839
840  // Insert (H, W) to (N, H, W, F).
841  Value combinedVal =
842  insert2DDataTo4D(builder, loc, matmulRetValue, args[0], NIter, FIter,
843  heightOffset, widthOffset, retRows, retCols,
844  /*loopNorFIdx=*/0,
845  /*loopCorFIdx=*/3, /*heightIdx=*/1,
846  /*widthIdx=*/2);
847
848  return {combinedVal};
849  };
850
851  int64_t tilwH = valueShape[2];
852  int64_t tileW = valueShape[3];
853  auto zeroIdx = rewriter.create<arith::ConstantIndexOp>(loc, 0);
854  auto tileHBound = rewriter.create<arith::ConstantIndexOp>(loc, tilwH);
855  auto tileWBound = rewriter.create<arith::ConstantIndexOp>(loc, tileW);
856  auto nUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueN);
857  auto fUpperBound = rewriter.create<arith::ConstantIndexOp>(loc, valueF);
858  auto oneStep = rewriter.create<arith::ConstantIndexOp>(loc, 1);
859  scf::LoopNest loops = scf::buildLoopNest(
860  rewriter, loc, {zeroIdx, zeroIdx, zeroIdx, zeroIdx},
861  {tileHBound, tileWBound, nUpperBound, fUpperBound},
862  {oneStep, oneStep, oneStep, oneStep}, {output}, buildBody);
863  return loops.results[0];
864 }
865
866 /// Create an empty tensor with alignedType and insert the value into the
867 /// created empty tensor with aligned size.
868 static Value padToAlignedTensor(RewriterBase &rewriter, Location loc,
869  Value value, ArrayRef<int64_t> alignedShape) {
870  auto valueType = cast<ShapedType>(value.getType());
871  Type elementType = valueType.getElementType();
872  auto alignedType = RankedTensorType::get(alignedShape, elementType);
874  loc, elementType, rewriter.getZeroAttr(elementType));
875
876  return linalg::makeComposedPadHighOp(rewriter, loc, alignedType, value,
878 }
879
880 /// Extract sub-tensor with extractedType from value.
881 static Value extractFromAlignedTensor(RewriterBase &rewriter, Location loc,
882  Value value,
883  RankedTensorType extractedType) {
884  OpFoldResult zeroIndex = rewriter.getIndexAttr(0);
885  OpFoldResult oneIndex = rewriter.getIndexAttr(1);
886  SmallVector<OpFoldResult, 4> offsets(4, zeroIndex);
887  SmallVector<OpFoldResult, 4> strides(4, oneIndex);
888
889  ArrayRef<int64_t> extractedShape = extractedType.getShape();
890  SmallVector<OpFoldResult> sizes =
891  getAsOpFoldResult(rewriter.getI64ArrayAttr(extractedShape));
892
893  return rewriter.create<tensor::ExtractSliceOp>(loc, extractedType, value,
894  offsets, sizes, strides);
895 }
896
897 /// Utility function to check all values in the attribute are 1.
898 static bool hasAllOneValues(DenseIntElementsAttr attr) {
899  return llvm::all_of(
900  attr, [](const APInt &element) { return element.getSExtValue() == 1; });
901 }
902
903 /// A helper function to convert linalg.conv_2d_nhwc_fhwc to
905 static FailureOr<Operation *>
907  int64_t m, int64_t r) {
908  Value input = convOp.getInputs()[0];
909  Value filter = convOp.getInputs()[1];
910  Value output = convOp.getOutputs()[0];
911  auto inputType = cast<ShapedType>(input.getType());
912  auto filterType = cast<ShapedType>(filter.getType());
913  auto outputType = cast<ShapedType>(output.getType());
914
915  if (!inputType.hasStaticShape())
916  return rewriter.notifyMatchFailure(convOp,
917  "expected a static shape for the input");
918
919  if (!filterType.hasStaticShape())
920  return rewriter.notifyMatchFailure(
921  convOp, "expected a static shape for the filter");
922
923  if (!hasAllOneValues(convOp.getDilations()))
924  return rewriter.notifyMatchFailure(convOp,
925  "expected all ones for dilations");
926
927  if (!hasAllOneValues(convOp.getStrides()))
928  return rewriter.notifyMatchFailure(convOp, "expected all ones for strides");
929
930  ArrayRef<int64_t> filterShape = filterType.getShape();
931  int64_t filterF = filterShape[0];
932  int64_t filterH = filterShape[1];
933  int64_t filterW = filterShape[2];
934  int64_t filterC = filterShape[3];
935  ArrayRef<int64_t> inputShape = inputType.getShape();
936  int64_t inputN = inputShape[0];
937  int64_t inputH = inputShape[1];
938  int64_t inputW = inputShape[2];
939  int64_t inputC = inputShape[3];
940  ArrayRef<int64_t> outputShape = outputType.getShape();
941  int64_t outputN = outputShape[0];
942  int64_t outputH = outputShape[1];
943  int64_t outputW = outputShape[2];
944  int64_t outputF = outputShape[3];
945
946  // Only support F(m x m, r x r), F(m x 1, r x 1) or F(1 x m, 1 x r).
947  bool isSupportedFilter = false;
948  if (filterH == filterW && filterH == r)
949  isSupportedFilter = true;
950  if (filterH == r && filterW == 1)
951  isSupportedFilter = true;
952  if (filterH == 1 && filterW == r)
953  isSupportedFilter = true;
954
955  if (!isSupportedFilter)
956  return rewriter.notifyMatchFailure(
957  convOp, "only support filter (r x r), (r x 1) or (1 x r)");
958
959  // Currently, we support (m, r) = (2, 3) or (4, 3) or (2, 5).
960  static const llvm::SmallVector<TransformMapKeyTy, 3> validConfigs = {
961  F_2_3, F_4_3, F_2_5};
962
963  TransformMapKeyTy key = {m, r};
964  auto it = std::find(validConfigs.begin(), validConfigs.end(), key);
965  // If we cannot find the constant transformation matrix, it means we do
966  // not support this configuration yet.
967  if (it == validConfigs.end())
968  return failure();
969
970  // All the criterias are satisfied. We can do Winograd Conv2D.
971  Location loc = convOp.getLoc();
972
973  // For F(m x 1, r x 1), we only need to do left side transform.
974  bool leftTransform = filterH != 1;
975  // For F(1 x m, 1 x r), we only need to do right side transform.
976  bool rightTransform = filterW != 1;
977  int64_t heightM = leftTransform ? m : 1;
978  int64_t widthM = rightTransform ? m : 1;
979  int64_t heightR = leftTransform ? r : 1;
980  int64_t widthR = rightTransform ? r : 1;
981
982  // --- Create operation for filter transform ---
983  Type filterElementType = filterType.getElementType();
984  int64_t alphaH = heightM + heightR - 1;
985  int64_t alphaW = widthM + widthR - 1;
986  int64_t tileH = llvm::divideCeilSigned(outputH, heightM);
987  int64_t tileW = llvm::divideCeilSigned(outputW, widthM);
988  auto retType = RankedTensorType::get({alphaH, alphaW, filterC, filterF},
989  filterElementType);
990  Value retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
991  filterElementType);
993  loc, retType, filter, retValue, m, r);
994
995  // --- Create operation for input transform ---
996
997  // When input size - (r - 1) is not aligned with output tile size, we need to
998  // pad the input data to create the full tiles as tiling.
999  Type inputElementType = inputType.getElementType();
1000  int64_t alignedInputH = tileH * heightM + (heightR - 1);
1001  int64_t alignedInputW = tileW * widthM + (widthR - 1);
1002  if (alignedInputH != inputH || alignedInputW != inputW) {
1003  input = padToAlignedTensor(rewriter, loc, input,
1004  {inputN, alignedInputH, alignedInputW, inputC});
1005  }
1006
1007  retType = RankedTensorType::get(
1008  {alphaH, alphaW, tileH, tileW, inputN, inputC}, inputElementType);
1009  retValue = rewriter.create<tensor::EmptyOp>(loc, retType.getShape(),
1010  inputElementType);
1012  loc, retType, input, retValue, m, r);
1013
1014  Type outputElementType = outputType.getElementType();
1015  Value matmulRet = matrixMultiply(rewriter, loc, transformedFilter,
1016  transformedInput, outputElementType);
1017
1018  // --- Create operation for output transform ---
1019
1020  // When output size is not aligned with output tile size, we need to pad the
1021  // output buffer to insert the full tiles after tiling.
1022  int64_t alignedOutputH = tileH * heightM;
1023  int64_t alignedOutputW = tileW * widthM;
1024  bool isOutputUnaligned =
1025  ((alignedOutputH != outputH) || (alignedOutputW != outputW));
1026  if (isOutputUnaligned) {
1027  auto alignedOutputType = RankedTensorType::get(
1028  {outputN, alignedOutputH, alignedOutputW, outputF}, outputElementType);
1029  output =
1031  outputType = alignedOutputType;
1032  }
1033
1035  loc, outputType, matmulRet, output, m, r);
1036
1037  // When output size is not aligned with output tile size, extract the
1038  // value from the padded buffer.
1039  if (isOutputUnaligned) {
1040  transformedOutput = extractFromAlignedTensor(
1041  rewriter, loc, transformedOutput,
1042  RankedTensorType::get({outputN, outputH, outputW, outputF},
1043  outputElementType));
1044  }
1045
1046  rewriter.replaceOp(convOp, transformedOutput);
1047
1048  return transformedOutput.getDefiningOp();
1049 }
1050
1051 /// A helper function to decompose linalg.winograd_filter_transform.
1052 FailureOr<Operation *>
1055  Location loc = op.getLoc();
1056  Value filter = op.getFilter();
1057  auto filterType = cast<ShapedType>(filter.getType());
1058  auto filterShape = filterType.getShape();
1059  int64_t filterH = filterShape[1];
1060  int64_t filterW = filterShape[2];
1061
1062  // For F(m x 1, r x 1), we only need to do left side transform.
1063  bool leftTransform = filterH != 1;
1064  // For F(1 x m, 1 x r), we only need to do right side transform.
1065  bool rightTransform = filterW != 1;
1066  Value transformedFilter =
1067  filterTransform(rewriter, loc, filter, op.getOutput(), op.getM(),
1068  op.getR(), leftTransform, rightTransform);
1069  if (!transformedFilter)
1070  return failure();
1071
1072  rewriter.replaceOp(op, transformedFilter);
1073
1074  return transformedFilter.getDefiningOp();
1075 }
1076
1077 /// A helper function to decompose linalg.winograd_input_transform.
1078 FailureOr<Operation *>
1081  Location loc = op.getLoc();
1082  Value input = op.getInput();
1083  auto inputType = cast<ShapedType>(input.getType());
1084  auto inputShape = inputType.getShape();
1085  int64_t inputH = inputShape[1];
1086  int64_t inputW = inputShape[2];
1087
1088  // For F(m x 1, r x 1), we only need to do left side transform.
1089  bool leftTransform = inputH != 1;
1090  // For F(1 x m, 1 x r), we only need to do right side transform.
1091  bool rightTransform = inputW != 1;
1092  Value transformedInput =
1093  inputTransform(rewriter, loc, op.getInput(), op.getOutput(), op.getM(),
1094  op.getR(), leftTransform, rightTransform);
1095  if (!transformedInput)
1096  return failure();
1097
1098  rewriter.replaceOp(op, transformedInput);
1099
1100  return transformedInput.getDefiningOp();
1101 }
1102
1103 /// A helper function to decompose linalg.winograd_output_transform.
1104 FailureOr<Operation *>
1107  Location loc = op.getLoc();
1108  Value value = op.getValue();
1109  auto valueType = cast<ShapedType>(value.getType());
1110  auto valueShape = valueType.getShape();
1111  int64_t valueH = valueShape[0];
1112  int64_t valueW = valueShape[1];
1113
1114  // For F(m x 1, r x 1), we only need to do left side transform.
1115  bool leftTransform = valueH != 1;
1116  // For F(1 x m, 1 x r), we only need to do right side transform.
1117  bool rightTransform = valueW != 1;
1118  Value transformedOutput =
1119  outputTransform(rewriter, loc, value, op.getOutput(), op.getM(),
1120  op.getR(), leftTransform, rightTransform);
1121  if (!transformedOutput)
1122  return failure();
1123
1124  rewriter.replaceOp(op, transformedOutput);
1125
1126  return transformedOutput.getDefiningOp();
1127 }
1128
1129 /// A rewrite pattern to decompose linalg.winograd_filter_transform operations.
1132 public:
1134
1136  PatternRewriter &rewriter) const override {
1138  }
1139 };
1140
1141 /// A rewrite pattern to decompose linalg.winograd_input_transform operations.
1144 public:
1146
1148  PatternRewriter &rewriter) const override {
1150  }
1151 };
1152
1153 /// A rewrite pattern to decompose linalg.winograd_output_transform operations.
1156 public:
1158
1160  PatternRewriter &rewriter) const override {
1162  }
1163 };
1164
1165 /// A rewrite pattern for Winograd Conv2D algorithm.
1167  : public OpRewritePattern<linalg::Conv2DNhwcFhwcOp> {
1168 public:
1170  WinogradConv2DNhwcFhwc(mlir::MLIRContext *context, int64_t m, int64_t r)
1171  : OpRewritePattern(context), m(m), r(r) {}
1172
1173  LogicalResult matchAndRewrite(linalg::Conv2DNhwcFhwcOp convOp,
1174  PatternRewriter &rewriter) const override {
1175  if (failed(winogradConv2DHelper(rewriter, convOp, m, r)))
1176  return failure();
1177
1178  return success();
1179  }
1180
1181 private:
1182  int64_t m;
1183  int64_t r;
1184 };
1185 } // end anonymous namespace
1186
1187 //===----------------------------------------------------------------------===//
1189  linalg::Conv2DNhwcFhwcOp op, int64_t m,
1190  int64_t r) {
1191  return winogradConv2DHelper(rewriter, op, m, r);
1192 }
1193
1194 FailureOr<Operation *>
1198 }
1199
1200 FailureOr<Operation *>
1204 }
1205
1206 FailureOr<Operation *>
1210 }
1211
1213  int64_t r) {
1214  MLIRContext *context = patterns.getContext();
1215  // TODO: Support more Conv2D data layout, e.g., conv_2d_nchw_fchw
1217 }
1218
1220  MLIRContext *context = patterns.getContext();
1221  patterns
1224 }
1225
1226 } // end namespace linalg
1227 } // end namespace mlir
int64_t scalarFactor
const float * table
int64_t cols
int64_t rows
static AffineMap get(MLIRContext *context)
Returns a zero result affine map with no dimensions or symbols: () -> ().
static DenseFPElementsAttr get(const ShapedType &type, Arg &&arg)
Get an instance of a DenseFPElementsAttr with the given arguments.
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:60
RewritePatternSet & insert(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
Definition: PatternMatch.h:931
MLIRContext * getContext() const
Definition: PatternMatch.h:823
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
Definition: PatternMatch.h:400
FailureOr< Operation * > winogradConv2D(RewriterBase &rewriter, linalg::Conv2DNhwcFhwcOp op, int64_t m, int64_t r)
Convert linalg.conv_2d_nhwc_fhwc to Winograd Conv2D algorithm F(m x m, r x r).
Value makeComposedPadHighOp(OpBuilder &b, Location loc, RankedTensorType type, Value source, Value pad, bool nofold)
Create a tensor::PadOp that pads source to the size of the statically sized type whose static sizes a...
Definition: Utils.cpp:192
static bool hasAllOneValues(DenseIntElementsAttr attr)
void populateWinogradConv2DPatterns(RewritePatternSet &patterns, int64_t m, int64_t r)
Patterns to apply Winograd Conv2D algorithm F(m x m, r x r).
@ Type
An inlay hint that for a type annotation.
LoopNest buildLoopNest(OpBuilder &builder, Location loc, ValueRange lbs, ValueRange ubs, ValueRange steps, ValueRange iterArgs, function_ref< ValueVector(OpBuilder &, Location, ValueRange, ValueRange)> bodyBuilder=nullptr)
Creates a perfect nest of "for" loops, i.e.
Definition: SCF.cpp:687
SmallVector< Value > ValueVector
An owning vector of values, handy to return from functions.
Definition: SCF.h:70
Include the generated interface declarations.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
OpFoldResult getAsOpFoldResult(Value val)
Given a value, try to extract a constant Attribute.
OpRewritePattern(MLIRContext *context, PatternBenefit benefit=1, ArrayRef< StringRef > generatedNames={})
Patterns must specify the root operation name they match against, and can also specify the benefit of...
Definition: PatternMatch.h:362
ValueVector results
Definition: SCF.h:74