VectorContractToAMXDotProduct.cpp

Go to the documentation of this file.

//===- VectorContractToAMXDotProduct.cpp ----------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/Dialect/X86/Transforms.h"
#include "mlir/Dialect/X86/Utils/X86Utils.h"
#include "mlir/Dialect/X86/X86Dialect.h"
 
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/Dominance.h"
#include "mlir/IR/PatternMatch.h"
#include "llvm/Support/Casting.h"
 
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
 
using namespace mlir;
using namespace mlir::vector;
using namespace mlir::x86;
 
namespace {
 
// Return true if the operation is a constant dense element
// attribute.
static bool isZeroVectorConstant(Operation *op) {
  auto constantOp = dyn_cast_if_present<arith::ConstantOp>(op);
  if (!constantOp)
    return false;
 
  if (!isa<VectorType>(constantOp.getType()))
    return false;
 
  auto denseAttr = dyn_cast<DenseElementsAttr>(constantOp.getValue());
  if (!denseAttr || !denseAttr.isSplat())
    return false;
 
  return isZeroIntegerOrFloat(denseAttr.getSplatValue<Attribute>());
}
 
// Recursively follows single-use values through scf.yield operations
// and returns the first non-yield user result in the contraction chain.
static Value contractionUsersAfterYield(Value v) {
  if (v.getNumUses() != 1)
    return nullptr;
 
  OpOperand &use = *v.use_begin();
  Operation *user = use.getOwner();
 
  if (!isa<scf::YieldOp>(user))
    return v;
 
  auto yield = cast<scf::YieldOp>(user);
  Operation *parent = yield->getParentOp();
  unsigned idx = use.getOperandNumber();
 
  return contractionUsersAfterYield(parent->getResult(idx));
}
 
// Function to collapse the last two dimension (k or n, and vnni) to help the
// amx.tile_load to correctly load the packed element type.
static Value collapseInnerDims(OpBuilder &builder, Location loc, Value input) {
  ShapedType inputType = cast<ShapedType>(input.getType());
  int64_t firstDimToCollapse = inputType.getRank() - 2;
 
  if (inputType.getRank() == 1)
    return input;
 
  SmallVector<ReassociationIndices> reassociation;
  for (int64_t i = 0; i < firstDimToCollapse; ++i)
    reassociation.push_back(ReassociationIndices{i});
 
  ReassociationIndices collapsedIndices;
  for (int64_t i = firstDimToCollapse; i < inputType.getRank(); ++i)
    collapsedIndices.push_back(i);
 
  reassociation.push_back(collapsedIndices);
  return memref::CollapseShapeOp::create(builder, loc, input, reassociation);
}
 
// Check if a vector.contract operand has a memref read source.
static bool isReadSrcMemref(Value operand) {
  Operation *defOp = operand.getDefiningOp();
  if (!defOp)
    return false;
 
  Value srcBuff;
  llvm::TypeSwitch<Operation *>(operand.getDefiningOp())
      .Case<TransferReadOp, LoadOp>(
          [&](auto readOp) { srcBuff = readOp.getOperand(0); });
 
  return srcBuff && isa<MemRefType>(srcBuff.getType());
}
 
// Check if a vector.contract operand reads whole VNNI groups: the memref's
// static innermost dim is the VNNI factor and the read starts at its origin.
static bool isVnniOperandRead(Value operand, int64_t vnni) {
  Operation *defOp = operand.getDefiningOp();
  if (!defOp)
    return false;
 
  Value srcBuff;
  OpFoldResult vnniIndex;
  llvm::TypeSwitch<Operation *>(defOp).Case<TransferReadOp, LoadOp>(
      [&](auto readOp) {
        srcBuff = readOp.getBase();
        vnniIndex = readOp.getIndices().back();
      });
 
  if (!srcBuff)
    return false;
  auto srcType = dyn_cast<MemRefType>(srcBuff.getType());
  return srcType && srcType.getRank() > 0 &&
         srcType.getShape().back() == vnni && isZeroInteger(vnniIndex);
}
 
// Replaces the indices of the two innermost dims of a VNNI operand by the
// index of the dim they collapse into.
static void collapseVnniIndices(OpBuilder &rewriter, Location loc,
                                SmallVectorImpl<Value> &indices, int64_t vnni) {
  Value inner = indices.pop_back_val();
  Value cVnni = arith::ConstantIndexOp::create(rewriter, loc, vnni);
  Value scaled = arith::MulIOp::create(rewriter, loc, indices.back(), cVnni);
  indices.back() = arith::AddIOp::create(rewriter, loc, scaled, inner);
}
 
// Get the MemRef source and offset index for the operands of
// vector.contract.
static FailureOr<std::pair<Value, SmallVector<Value>>>
getSrcIndxValue(OpBuilder &rewriter, Location loc, Value operand,
                bool isNotAcc) {
  Operation *defOp = operand.getDefiningOp();
  if (!defOp)
    return failure();
 
  Value srcBuff;
  SmallVector<OpFoldResult> indexVals;
  llvm::TypeSwitch<Operation *>(operand.getDefiningOp())
      .Case<TransferReadOp, LoadOp>([&](auto readOp) {
        indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(),
                                              readOp.getIndices().end());
        srcBuff = readOp.getOperand(0);
      });
 
  if (!srcBuff || !isa<MemRefType>(srcBuff.getType()))
    return failure();
 
  SmallVector<Value> indices;
  indices.reserve(indexVals.size());
 
  for (OpFoldResult ofr : indexVals) {
    indices.push_back(
        mlir::getValueOrCreateConstantIndexOp(rewriter, loc, ofr));
  }
 
  if (isNotAcc) {
    collapseVnniIndices(rewriter, loc, indices,
                        cast<MemRefType>(srcBuff.getType()).getShape().back());
    srcBuff = collapseInnerDims(rewriter, loc, srcBuff);
  }
 
  return std::make_pair(srcBuff, indices);
}
 
// Function to validate the loop step value.
static LogicalResult validateLoopStep(OpBuilder &rewriter, Value step,
                                      int64_t value) {
 
  auto cst = step.getDefiningOp<arith::ConstantIndexOp>();
  if (!cst)
    return failure();
 
  if (cst.value() != value && cst.value() != 1)
    return failure();
 
  return success();
}
 
// Function to validate the vector.contract operation.
static LogicalResult validateContractOps(OpBuilder &rewriter,
                                         vector::ContractionOp contractOp,
                                         unsigned int blockingFactor,
                                         Value srcBuffLhs, Value srcBuffRhs,
                                         bool srcValidate) {
 
  if (srcValidate) {
    // Get the MemRef buffer of LHS operand.
    auto srcIndxLhs = getSrcIndxValue(rewriter, contractOp.getLoc(),
                                      contractOp.getLhs(), false);
    if (failed(srcIndxLhs))
      return failure();
    auto [buffLhs, indicesLhs] = *srcIndxLhs;
 
    // Get the MemRef buffer of RHS operand.
    auto srcIndxRhs = getSrcIndxValue(rewriter, contractOp.getLoc(),
                                      contractOp.getRhs(), false);
    if (failed(srcIndxRhs))
      return failure();
    auto [buffRhs, indicesRhs] = *srcIndxRhs;
 
    // Return failure if the Memref buff didn't match.
    if (buffLhs != srcBuffLhs)
      return failure();
 
    if (buffRhs != srcBuffRhs)
      return failure();
  }
 
  if (!contractionUsersAfterYield(contractOp.getResult()))
    return failure();
 
  VectorType accTy = dyn_cast<VectorType>(contractOp.getAccType());
  if (!accTy)
    return failure();
 
  // The Accumulator dims should be 16 or 1. Like <1x16x16> or <16x16>.
  ArrayRef<int64_t> accShape = accTy.getShape();
  SmallVector<int64_t> nonUnitDimAcc;
  llvm::copy_if(accShape, std::back_inserter(nonUnitDimAcc),
                [](int64_t dim) { return (dim != 16 && dim != 1); });
 
  if (nonUnitDimAcc.size() != 0)
    return failure();
 
  // The LHS dims should be 16 or vnni or 1. Like <1x16x16x2> or
  // <16x16x4>. The vnni dims should be 2 or 4.
  VectorType lhsTy = contractOp.getLhsType();
  ArrayRef<int64_t> lhsShape = lhsTy.getShape();
  SmallVector<int64_t> nonUnitDimLhs;
  llvm::copy_if(lhsShape, std::back_inserter(nonUnitDimLhs),
                [](int64_t dim) { return (dim != 16 && dim != 1); });
 
  if (nonUnitDimLhs.size() != 1)
    return failure();
 
  if (nonUnitDimLhs[0] != blockingFactor)
    return failure();
 
  // The RHS dims should be 16 or vnni or 1. Like <1x16x16x2> or
  // <16x16x4>. The vnni dims should be 2 or 4.
  VectorType rhsTy = contractOp.getRhsType();
  ArrayRef<int64_t> rhsShape = rhsTy.getShape();
  llvm::SmallVector<int64_t> nonUnitDimRhs;
  llvm::copy_if(rhsShape, std::back_inserter(nonUnitDimRhs),
                [](int64_t dim) { return (dim != 16 && dim != 1); });
 
  if (nonUnitDimRhs.size() != 1)
    return failure();
 
  if (nonUnitDimRhs[0] != blockingFactor)
    return failure();
 
  return success();
}
 
// Traces an accumulator back through the iter_args of the given loops and
// returns the first value that is not one of them, e.g., the init value of the
// outermost loop or a value computed within the loops.
static Value getLoopNestInit(Value acc, ArrayRef<scf::ForOp> loops) {
  while (auto arg = dyn_cast<BlockArgument>(acc)) {
    auto loop = dyn_cast_if_present<scf::ForOp>(arg.getOwner()->getParentOp());
    if (!loop || !llvm::is_contained(loops, loop))
      break;
    OpOperand *init = loop.getTiedLoopInit(arg);
    if (!init)
      break;
    acc = init->get();
  }
  return acc;
}
 
// Returns the loop index position to get mapped during the
// MemRef type clone.
static unsigned getIndexPosition(Value operand, scf::ForOp loop) {
  Value iv = loop.getInductionVar();
 
  Value srcBuff;
  llvm::TypeSwitch<Operation *>(operand.getDefiningOp())
      .Case<TransferReadOp, LoadOp>(
          [&](auto readOp) { srcBuff = readOp.getOperand(0); });
 
  auto subview = srcBuff.getDefiningOp<memref::SubViewOp>();
  if (!subview)
    return 0;
 
  auto offsets = subview.getOffsets();
 
  for (auto it : llvm::enumerate(offsets)) {
    if (it.value() == iv)
      return it.index();
  }
 
  return 0;
}
 
// Creates amx.tile_loads.
static amx::TileLoadOp createTileLoads(OpBuilder &rewriter, Location loc,
                                       Value operand, Value mat, Type ipType,
                                       unsigned int offset, bool isVnni) {
 
  auto srcIndx = getSrcIndxValue(rewriter, loc, operand, false);
  auto [srcBuff, indices] = *srcIndx;
  if (isVnni)
    collapseVnniIndices(rewriter, loc, indices,
                        cast<MemRefType>(srcBuff.getType()).getShape().back());
 
  amx::TileType tileType = amx::TileType::get({16, (16 * offset)}, ipType);
  return amx::TileLoadOp::create(rewriter, loc, tileType, mat, indices);
}
 
static void performShuffle(OpBuilder &rewriter, Location loc, Value matB,
                           Type ipType, unsigned int offset, Value packedBuffer,
                           Value indxToStoreInBuffer) {
 
  Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
  Value c16 = arith::ConstantIndexOp::create(rewriter, loc, 16);
  SmallVector<Value> subviewOffset(
      llvm::cast<MemRefType>(matB.getType()).getRank(), c0);
 
  Value cStep = arith::ConstantIndexOp::create(rewriter, loc, offset);
  Value cBound = arith::ConstantIndexOp::create(rewriter, loc, (16 * offset));
  Value offsetIndx =
      arith::ConstantIndexOp::create(rewriter, loc, (offset / 2));
 
  scf::ForOp::create(
      rewriter, loc, c0, cBound, cStep, ValueRange{},
      [&](OpBuilder &nestedBuilder, Location loc, Value iv,
          ValueRange iterArgs) {
        subviewOffset[subviewOffset.size() - 2] = iv;
 
        // Retrieve two rows of vector (32) for int8 and f8 type. For bf16,
        // retrieve one row of vector (32).
        auto vectorType = VectorType::get({2, (16 * (offset / 2))}, ipType);
        if (ipType.isBF16())
          vectorType = VectorType::get((16 * offset), ipType);
 
        int64_t srcRank = (dyn_cast<ShapedType>(matB.getType())).getRank();
        Value padding = ub::PoisonOp::create(rewriter, loc, ipType);
        auto map = AffineMap::getMinorIdentityMap(srcRank, vectorType.getRank(),
                                                  rewriter.getContext());
        SmallVector<bool> inBounds(vectorType.getRank(), true);
        Value vec1 = vector::TransferReadOp::create(
            rewriter, loc, vectorType, matB, ValueRange(subviewOffset), padding,
            map, inBounds);
 
        if (!ipType.isBF16())
          vec1 = vector::ShapeCastOp::create(
              rewriter, loc, VectorType::get((16 * offset), ipType), vec1);
 
        // Increment the iv by 1 or 2 based on the type to load the next 32/64
        // elements
        Value incIV = arith::AddIOp::create(rewriter, loc, offsetIndx, iv);
        subviewOffset[subviewOffset.size() - 2] = incIV;
 
        Value vec2 = vector::TransferReadOp::create(
            rewriter, loc, vectorType, matB, ValueRange(subviewOffset), padding,
            map, inBounds);
        if (!ipType.isBF16())
          vec2 = vector::ShapeCastOp::create(
              rewriter, loc, VectorType::get((16 * offset), ipType), vec2);
 
        vector::ShuffleOp writeVal1;
        vector::ShuffleOp writeVal2;
 
        if (ipType.isBF16()) {
 
          writeVal1 = vector::ShuffleOp::create(
              rewriter, loc, VectorType::get({(16 * offset)}, ipType), vec1,
              vec2,
              ArrayRef<int64_t>{0,  32, 1,  33, 2,  34, 3,  35, 8,  40, 9,
                                41, 10, 42, 11, 43, 16, 48, 17, 49, 18, 50,
                                19, 51, 24, 56, 25, 57, 26, 58, 27, 59});
 
          writeVal2 = vector::ShuffleOp::create(
              rewriter, loc, VectorType::get({(16 * offset)}, ipType), vec1,
              vec2,
              ArrayRef<int64_t>{4,  36, 5,  37, 6,  38, 7,  39, 12, 44, 13,
                                45, 14, 46, 15, 47, 20, 52, 21, 53, 22, 54,
                                23, 55, 28, 60, 29, 61, 30, 62, 31, 63});
        }
 
        if (ipType.isSignlessInteger(8) || ipType.isF8E5M2() ||
            ipType.isF8E4M3FN()) {
 
          writeVal1 = vector::ShuffleOp::create(
              rewriter, loc, VectorType::get({(16 * offset)}, ipType), vec1,
              vec2,
              ArrayRef<int64_t>{
                  0,   32,  64, 96,  1,   33,  65,  97,  2,   34,  66,  98, 3,
                  35,  67,  99, 8,   40,  72,  104, 9,   41,  73,  105, 10, 42,
                  74,  106, 11, 43,  75,  107, 16,  48,  80,  112, 17,  49, 81,
                  113, 18,  50, 82,  114, 19,  51,  83,  115, 24,  56,  88, 120,
                  25,  57,  89, 121, 26,  58,  90,  122, 27,  59,  91,  123});
 
          writeVal2 = vector::ShuffleOp::create(
              rewriter, loc, VectorType::get({(16 * offset)}, ipType), vec1,
              vec2,
              ArrayRef<int64_t>{
                  4,  36,  68, 100, 5,  37,  69, 101, 6,  38,  70, 102, 7,  39,
                  71, 103, 12, 44,  76, 108, 13, 45,  77, 109, 14, 46,  78, 110,
                  15, 47,  79, 111, 20, 52,  84, 116, 21, 53,  85, 117, 22, 54,
                  86, 118, 23, 55,  87, 119, 28, 60,  92, 124, 29, 61,  93, 125,
                  30, 62,  94, 126, 31, 63,  95, 127});
        }
 
        // iv to store the shuffled elements
        Value ivShuff1 = arith::DivUIOp::create(rewriter, loc, iv, cStep);
        Value ivShuff2 = arith::AddIOp::create(rewriter, loc, ivShuff1, c16);
 
        vector::StoreOp::create(rewriter, loc, writeVal1, packedBuffer,
                                ValueRange{indxToStoreInBuffer, ivShuff1, c0});
        vector::StoreOp::create(rewriter, loc, writeVal2, packedBuffer,
                                ValueRange{indxToStoreInBuffer, ivShuff2, c0});
 
        scf::YieldOp::create(nestedBuilder, loc);
      });
}
 
static llvm::DenseMap<Operation *, amx::TileLoadOp>
packInputs(OpBuilder &rewriter, Location loc,
           SmallVector<vector::ContractionOp> ops, Value matB, Type ipType,
           unsigned int offset, Value packedBuffer, bool pack,
           Value indxToStoreInBuffer, Value indxToLoadFromMatB) {
 
  llvm::DenseMap<Operation *, amx::TileLoadOp> readsToTileLoads;
  Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
  Value c16 = arith::ConstantIndexOp::create(rewriter, loc, 16);
 
  for (size_t j = 0; j < ops.size(); j++) {
    for (size_t i = 0; i < ops.size(); i++) {
 
      if (i != j && validatePairVectorContract(ops[j], ops[i], true, 16)) {
 
        Operation *readOpRhs = ops[j].getRhs().getDefiningOp();
        auto itRhs = readsToTileLoads.find(readOpRhs);
        if (itRhs != readsToTileLoads.end()) {
          continue;
        }
 
        if (pack) {
          performShuffle(rewriter, loc, matB, ipType, offset, packedBuffer,
                         indxToStoreInBuffer);
        }
 
        amx::TileType tileType =
            amx::TileType::get({16, (16 * offset)}, ipType);
        auto loadRow1 =
            amx::TileLoadOp::create(rewriter, loc, tileType, packedBuffer,
                                    ValueRange{indxToLoadFromMatB, c0, c0});
 
        auto loadRow2 =
            amx::TileLoadOp::create(rewriter, loc, tileType, packedBuffer,
                                    ValueRange{indxToLoadFromMatB, c16, c0});
 
        readsToTileLoads.try_emplace(readOpRhs, loadRow1);
        readsToTileLoads.try_emplace(ops[i].getRhs().getDefiningOp(), loadRow2);
      }
    }
  }
 
  return readsToTileLoads;
}
 
// Creates tiled amx dot-products.
static SmallVector<Value>
createTiledDp(OpBuilder &rewriter, Location loc,
              SmallVector<vector::ContractionOp> ops, Value matA, Value matB,
              Type ipType, Type opType, ValueRange accIterArgs,
              unsigned int offset, bool isVnni, Value packedBuffer, bool pack,
              Value indxToStoreInBuffer, Value indxToLoadFromMatB) {
 
  if (isVnni) {
    matA = collapseInnerDims(rewriter, loc, matA);
    matB = collapseInnerDims(rewriter, loc, matB);
  }
 
  SmallVector<Value> accumulators;
  // Stores the amx.tile_load operation vs it's equivalent vector tranfer_read
  // or load operations.
  llvm::DenseMap<Operation *, amx::TileLoadOp> readsToTileLoads;
 
  // function call to online pack the input  B matrix
  if (!isVnni) {
    readsToTileLoads =
        packInputs(rewriter, loc, ops, matB, ipType, offset, packedBuffer, pack,
                   indxToStoreInBuffer, indxToLoadFromMatB);
  }
 
  // Iterate over the contraction operations and compute the tiled dot-product.
  for (size_t i = 0; i < ops.size(); i++) {
 
    Operation *readOpLhs = ops[i].getLhs().getDefiningOp();
    amx::TileLoadOp tilesLhs;
    auto itLhs = readsToTileLoads.find(readOpLhs);
    if (itLhs != readsToTileLoads.end()) {
      tilesLhs = itLhs->second;
    } else {
      tilesLhs = createTileLoads(rewriter, loc, ops[i].getLhs(), matA, ipType,
                                 offset, isVnni);
      readsToTileLoads.try_emplace(readOpLhs, tilesLhs);
    }
 
    Operation *readOpRhs = ops[i].getRhs().getDefiningOp();
    amx::TileLoadOp tilesRhs;
    auto itRhs = readsToTileLoads.find(readOpRhs);
    if (itRhs != readsToTileLoads.end()) {
      tilesRhs = itRhs->second;
    } else {
      tilesRhs = createTileLoads(rewriter, loc, ops[i].getRhs(), matB, ipType,
                                 offset, isVnni);
      readsToTileLoads.try_emplace(readOpRhs, tilesRhs);
    }
 
    auto accTileType = amx::TileType::get({16, 16}, opType);
 
    Value dp;
    if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN())
      dp = amx::TileMulFOp::create(rewriter, loc, accTileType, tilesLhs,
                                   tilesRhs, accIterArgs[i]);
 
    if (ipType.isSignlessInteger(8))
      dp = amx::TileMulIOp::create(rewriter, loc, accTileType, tilesLhs,
                                   tilesRhs, accIterArgs[i]);
 
    accumulators.push_back(dp);
  }
  return accumulators;
}
 
static SmallVector<Value> createTileZeros(OpBuilder &rewriter, Location loc,
                                          Type opType, scf::ForOp outerLoop,
                                          int64_t size) {
  OpBuilder::InsertionGuard guard(rewriter);
  rewriter.setInsertionPoint(outerLoop);
 
  SmallVector<Value> loopItrArgs;
  auto zeroTileType = amx::TileType::get({16, 16}, opType);
 
  for (int i = 0; i < size; i++) {
    auto zeroTile = amx::TileZeroOp::create(rewriter, loc, zeroTileType);
    loopItrArgs.push_back(zeroTile);
  }
  return loopItrArgs;
}
 
// Find memref source and indices of the accumulator tile for an AMX
// contraction. Returns nullopt if the access is not suitable for AMX tile
// load/store.
static std::optional<std::pair<Value, SmallVector<Value>>>
getAccTileAccess(Operation *op) {
  if (!op)
    return std::nullopt;
 
  Value base;
  VectorType vecTy;
  ValueRange indices;
  if (auto transfer = dyn_cast<VectorTransferOpInterface>(op)) {
    if (transfer.getMask() || transfer.hasOutOfBoundsDim() ||
        !transfer.getPermutationMap().isMinorIdentity())
      return std::nullopt;
    base = transfer.getBase();
    vecTy = transfer.getVectorType();
    indices = transfer.getIndices();
  } else if (auto load = dyn_cast<vector::LoadOp>(op)) {
    base = load.getBase();
    vecTy = load.getVectorType();
    indices = load.getIndices();
  } else if (auto store = dyn_cast<vector::StoreOp>(op)) {
    base = store.getBase();
    vecTy = store.getVectorType();
    indices = store.getIndices();
  } else {
    return std::nullopt;
  }
 
  auto memrefTy = dyn_cast<MemRefType>(base.getType());
  ArrayRef<int64_t> shape = vecTy.getShape();
  if (!memrefTy || memrefTy.getRank() < 2 || !memrefTy.isLastDimUnitStride() ||
      shape.size() < 2 || shape.end()[-1] != 16 || shape.end()[-2] != 16 ||
      llvm::any_of(shape.drop_back(2), [](int64_t d) { return d != 1; }))
    return std::nullopt;
 
  return std::make_pair(base, SmallVector<Value>(indices));
}
 
// Returns the accumulator read and result write for each contraction if the
// accumulator can be updated in place through direct AMX tile accesses.
// If writeOnly is true, only the result writes are considered, and the read
// accesses are ignored.
static std::pair<std::optional<SmallVector<Operation *>>,
                 std::optional<SmallVector<Operation *>>>
getInPlaceAccAccesses(ArrayRef<vector::ContractionOp> ops, bool writeOnly) {
  SmallVector<Operation *> reads, writes;
  for (vector::ContractionOp op : ops) {
    Value result = contractionUsersAfterYield(op.getResult());
    if (!result || !result.hasOneUse())
      return {std::nullopt, std::nullopt};
    Operation *write = *result.user_begin();
    if (!isa<vector::TransferWriteOp, vector::StoreOp>(write) ||
        write->getOperand(0) != result || !getAccTileAccess(write))
      return {std::nullopt, std::nullopt};
    writes.push_back(write);
 
    if (writeOnly)
      continue;
 
    Operation *read = traceToVectorReadLikeParentOperation(op.getAcc());
    if (!read || !getAccTileAccess(read))
      return {std::nullopt, std::nullopt};
    reads.push_back(read);
  }
 
  if (writeOnly)
    return {std::nullopt, writes};
  return std::make_pair(reads, writes);
}
 
// Loads the initial accumulator tiles before the given loop.
static SmallVector<Value> createAccTileLoads(OpBuilder &rewriter, Location loc,
                                             Type opType, scf::ForOp loop,
                                             ArrayRef<Operation *> reads) {
  OpBuilder::InsertionGuard guard(rewriter);
  rewriter.setInsertionPoint(loop);
  auto tileType = amx::TileType::get({16, 16}, opType);
  SmallVector<Value> tiles;
  for (Operation *read : reads) {
    auto [base, indices] = *getAccTileAccess(read);
    tiles.push_back(
        amx::TileLoadOp::create(rewriter, loc, tileType, base, indices));
  }
  return tiles;
}
 
static Value getIndxToLoadStoreFromPckBuffer(OpBuilder &rewriter, Location loc,
                                             Value ivInnerLoop,
                                             Value ivOuterLoop,
                                             bool isInnerLoopUBHasOddQuot,
                                             bool isInnerLoopUBLarger,
                                             bool pack, Value blockStride) {
 
  Value c2 = arith::ConstantIndexOp::create(rewriter, loc, 2);
 
  // `blockStride` is the reduction (K) loop step, i.e. the amount by which the
  // induction variable advances for one K-block. Dividing the induction value
  // by it yields the K-block index regardless of whether the loop counts
  // K-elements (step == 16*blockingFactor) or pre-blocked K-tiles (step == 1).
  Value quotientInnerLoop =
      arith::DivUIOp::create(rewriter, loc, ivInnerLoop, blockStride);
  Value remInnerLoop = arith::RemUIOp::create(
      rewriter, loc, rewriter.getIndexType(), quotientInnerLoop, c2);
 
  if (!isInnerLoopUBLarger && !pack) {
    remInnerLoop = arith::RemUIOp::create(
        rewriter, loc, rewriter.getIndexType(), ivOuterLoop, c2);
  }
 
  if (isInnerLoopUBHasOddQuot) {
    auto remOuterLoop = arith::RemUIOp::create(
        rewriter, loc, rewriter.getIndexType(), ivOuterLoop, c2);
    auto remAdd = arith::AddIOp::create(rewriter, loc, rewriter.getIndexType(),
                                        remInnerLoop, remOuterLoop);
    remInnerLoop = arith::RemUIOp::create(rewriter, loc,
                                          rewriter.getIndexType(), remAdd, c2);
  }
 
  return remInnerLoop;
}
 
static scf::ForOp
createLoops(OpBuilder &rewriter, Location loc, Value lowerBound,
            Value upperBound, Value step, SmallVector<Value> loopItrArgs,
            Type ipType, Type opType, unsigned int blockingFactor, bool isVnni,
            Operation *vectorOpLhs, Operation *vectorOpRhs,
            vector::ContractionOp contractOp, scf::ForOp outerLoop,
            scf::ForOp innerLoop, SmallVector<vector::ContractionOp> ops,
            Value ivOuterLoop, Value packedBuffer, bool pack,
            arith::ConstantIndexOp innerLoopIndex, bool isInnerLoopUBLarger,
            bool isInnerLoopUBHasOddQuot) {
 
  Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
  Value c1 = arith::ConstantIndexOp::create(rewriter, loc, 1);
  Value c2 = arith::ConstantIndexOp::create(rewriter, loc, 2);
 
  int64_t offset = 16 * blockingFactor;
  if (auto cst = step.getDefiningOp<arith::ConstantIndexOp>())
    offset = cst.value();
 
  auto newLoop = scf::ForOp::create(
      rewriter, loc, lowerBound, upperBound, step, loopItrArgs,
      [&](OpBuilder &rewriterNewInnerLoop, Location locNewInnerLoop,
          Value ivNewInnerLoop, ValueRange iterArgsNewInnerLoop) {
        IRMapping mapping;
        if (outerLoop)
          mapping.map(vectorOpLhs->getOperand(
                          getIndexPosition(contractOp.getLhs(), outerLoop) + 1),
                      ivOuterLoop);
 
        mapping.map(vectorOpLhs->getOperand(
                        getIndexPosition(contractOp.getLhs(), innerLoop) + 1),
                    ivNewInnerLoop);
        auto lhsClone = rewriterNewInnerLoop.clone(*vectorOpLhs, mapping);
 
        Value indxToStoreInBuffer = c0;
        Value indxToLoadFromBuffer = c0;
        if (!isVnni) {
          if (outerLoop) {
            if (innerLoopIndex.value() == 0) {
              if (pack) {
                ivNewInnerLoop = c0;
                ivOuterLoop = arith::AddIOp::create(rewriter, locNewInnerLoop,
                                                    c1, ivOuterLoop);
 
                if (!isInnerLoopUBLarger || isInnerLoopUBHasOddQuot) {
                  indxToStoreInBuffer = arith::RemUIOp::create(
                      rewriter, locNewInnerLoop, rewriter.getIndexType(),
                      ivOuterLoop, c2);
                }
 
                Value indxToLoadFromMatB = arith::AddIOp::create(
                    rewriter, loc, indxToStoreInBuffer, c1);
                indxToLoadFromBuffer = arith::RemUIOp::create(
                    rewriter, loc, rewriter.getIndexType(), indxToLoadFromMatB,
                    c2);
              }
 
            } else {
              Value nLoadIndx = arith::ConstantIndexOp::create(
                  rewriter, locNewInnerLoop, offset);
              ivNewInnerLoop = arith::AddIOp::create(rewriter, locNewInnerLoop,
                                                     nLoadIndx, ivNewInnerLoop);
              indxToStoreInBuffer = getIndxToLoadStoreFromPckBuffer(
                  rewriter, loc, ivNewInnerLoop, ivOuterLoop,
                  isInnerLoopUBHasOddQuot, isInnerLoopUBLarger, pack, step);
              Value indxToLoadFromMatB =
                  arith::AddIOp::create(rewriter, loc, indxToStoreInBuffer, c1);
              indxToLoadFromBuffer =
                  arith::RemUIOp::create(rewriter, loc, rewriter.getIndexType(),
                                         indxToLoadFromMatB, c2);
            }
          } else {
            if (pack) {
              Value nLoadIndx = arith::ConstantIndexOp::create(
                  rewriter, locNewInnerLoop, offset);
              ivNewInnerLoop = arith::AddIOp::create(rewriter, locNewInnerLoop,
                                                     nLoadIndx, ivNewInnerLoop);
              Value quotient_K = arith::DivUIOp::create(
                  rewriter, loc, ivNewInnerLoop, nLoadIndx);
              indxToStoreInBuffer = arith::RemUIOp::create(
                  rewriter, loc, rewriter.getIndexType(), quotient_K, c2);
 
              Value indxToLoadFromMatB =
                  arith::AddIOp::create(rewriter, loc, indxToStoreInBuffer, c1);
              indxToLoadFromBuffer =
                  arith::RemUIOp::create(rewriter, loc, rewriter.getIndexType(),
                                         indxToLoadFromMatB, c2);
            }
          }
        }
        IRMapping rhsMapping;
 
        Value matB;
        Operation *rhsOp = vectorOpRhs;
 
        // Clone for the subview type operations
        if (rhsOp->getNumOperands() > 0) {
 
          if (outerLoop) {
            int64_t outerPos = getIndexPosition(contractOp.getRhs(), outerLoop);
 
            if (outerPos >= 0) {
              unsigned operandIdx = static_cast<unsigned>(outerPos + 1);
 
              if (operandIdx < rhsOp->getNumOperands())
                rhsMapping.map(rhsOp->getOperand(operandIdx), ivOuterLoop);
            }
          }
 
          int64_t innerPos = getIndexPosition(contractOp.getRhs(), innerLoop);
 
          if (innerPos >= 0) {
            unsigned operandIdx = static_cast<unsigned>(innerPos + 1);
 
            if (operandIdx < rhsOp->getNumOperands())
              rhsMapping.map(rhsOp->getOperand(operandIdx), ivNewInnerLoop);
          }
 
          auto rhsClone = rewriterNewInnerLoop.clone(*rhsOp, rhsMapping);
          matB = rhsClone->getResult(0);
 
        } else {
          // The mat B is of kind 'memref.get_global @__constant'
          matB = rhsOp->getResult(0);
        }
 
        if (!isVnni) {
          if (outerLoop) {
            if (!pack) {
              matB = Value();
              indxToLoadFromBuffer = c0;
              // Use the real spill-block induction value (== spillInnerLoop)
              // together with the loop step so the computed ping-pong slot
              // matches the prefetch store side for any number of register
              // blocks, including odd counts (e.g. 96 = 3 blocks). Passing a
              // constant here mis-parities the slot for odd block counts.
              indxToLoadFromBuffer = getIndxToLoadStoreFromPckBuffer(
                  rewriter, loc, ivNewInnerLoop, ivOuterLoop,
                  isInnerLoopUBHasOddQuot, isInnerLoopUBLarger, pack, step);
            }
          } else {
            if (!pack) {
              Value nLoadIndx = arith::ConstantIndexOp::create(
                  rewriter, locNewInnerLoop, offset);
              matB = Value();
              Value quotient_K = arith::DivUIOp::create(
                  rewriter, loc, ivNewInnerLoop, nLoadIndx);
              indxToLoadFromBuffer = arith::RemUIOp::create(
                  rewriter, loc, rewriter.getIndexType(), quotient_K, c2);
            }
          }
        }
        // compute tiled dot-product
        SmallVector<Value> accumulators = createTiledDp(
            rewriter, locNewInnerLoop, ops, lhsClone->getResult(0), matB,
            ipType, opType, iterArgsNewInnerLoop, blockingFactor, isVnni,
            packedBuffer, pack, indxToStoreInBuffer, indxToLoadFromBuffer);
 
        scf::YieldOp::create(rewriterNewInnerLoop, locNewInnerLoop,
                             accumulators);
      });
 
  return newLoop;
}
 
// Implements tiled dot-product operation for a vector.contract operation or a
// sequence of vector.contracts inside the reduction loops.
//
// For example:
// Case 1: register blocked vector.contract with prepacked input
// ```
//   vector.transfer_read %arg0 {{.}*} : memref<16x32x4xi8>, vector<16x16x4xi8>
//   vector.transfer_read %arg1 {{.}*} : memref<16x32x4xi8>, vector<16x16x4xi8>
//   vector.contract <16x16x4xi8>, <16x16x4xi8> into <16x16xi32>
//   vector.transfer_write arg2 {{.}*} : vector<16x16xi32>, memref<32x32xi32>
// ```
// to
// ```
//   amx.tile_load %arg0 {{.}*} : memref<16x32x4xi8> into !amx.tile<16x64xi8>
//   amx.tile_load %arg1 {{.}*} : memref<16x32x4xi8> into !amx.tile<16x64xi8>
//   amx.tile_muli !amx.tile<16x64xi8> -> !amx.tile<16x16xi32>
//   amx.tile_store %arg2{{.}*} : memref<32x32xi32>, !amx.tile<16x16xi32>
// ```
//
//
// Case2: vector.contract with register blocked
//
// Output IR with online packing (with s/w pipeline advantage):
// s/w pipeline: load, pack to VNNI, and store the B sub matrix
// of the 0th batch-reduce and K iteration.
// scf.for (0 to 31) {
//  - load 0th and 1st  vector<32xbf16>, pack into VNNI, store the
//  first shuffle in 0th and 2nd shuffle in 16th index of the
//  buffer.
// }
// scf.for (0 to br-2) { batch-reduce loop
//   scf.for (0 to k-2) { K loop
//  - load A matrix
//  - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub
// matrix   for the next K loop iteration   (c) load VNNI pack B matrix of K
// iteration from the buffer  (d) compute the tiled dot-product
//   }
//   Last iteration of the the K Loop (k-1) {
//      - load A matrix
//      - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub
//      matrix for the next batch-reduce + K loop iteration (c) load VNNI pack B
//      matrix of K iteration from the buffer (d) compute the tiled dot-product
//   }
// }
// Last iteration of the batch-reduce loop (br-1) {
//   scf.for (0 to k-2) { K loop
//      - load A matrix
//      - scf.loop for s/w pipeline: load, pack to VNNI, and store the B sub
//      matrix for the next K loop iteration (c) load VNNI pack B matrix of K
//      iteration from the buffer (d) compute the tiled dot-product
//   }
//   Last iteration of the the K Loop (k-1) {
//      - load A matrix
//      - load VNNI pack B matrix of K iteration from the buffer
//      - compute the tiled dot-product
//   }
// }
//
// scf.for (0 to M)
//   scf.for (0 to N)
//     - Load the ith and i+1th acc
//     - Shuffle them as we packed using vpunpack
//     - Load C matrix and do arith.add with the shuffle
//     - Store back into C matrix
struct VectorContractToAMXDotProduct
    : public OpRewritePattern<vector::ContractionOp> {
  using OpRewritePattern<vector::ContractionOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(vector::ContractionOp contractOp,
                                PatternRewriter &rewriter) const override {
 
    if (contractOp.getKind() != vector::CombiningKind::ADD)
      return rewriter.notifyMatchFailure(contractOp,
                                         "Expects add combining kind.");
 
    unsigned int blockingFactor =
        contractOp.getLhsType().getElementType().isBF16() ? 2 : 4;
    bool isVnni =
        isInVnniLayout(contractOp.getOperation(),
                       contractOp.getIndexingMapsArray(), blockingFactor);
 
    VectorType lhsTy = contractOp.getLhsType();
    if (!lhsTy.getElementType().isBF16() &&
        !lhsTy.getElementType().isSignlessInteger(8) &&
        !lhsTy.getElementType().isF8E4M3FN() &&
        !lhsTy.getElementType().isF8E5M2())
      return rewriter.notifyMatchFailure(
          contractOp, "Only BF16/Int8/F8 lowering is supported.");
 
    if (lhsTy.getElementType() != contractOp.getRhsType().getElementType())
      return rewriter.notifyMatchFailure(
          contractOp, "Contraction should have same lhs and rhs type.");
 
    VectorType accTy = dyn_cast<VectorType>(contractOp.getAccType());
    if (!accTy)
      return rewriter.notifyMatchFailure(contractOp, "Wrong accmulator type.");
 
    if (((lhsTy.getElementType().isBF16() ||
          lhsTy.getElementType().isF8E4M3FN() ||
          lhsTy.getElementType().isF8E5M2()) &&
         !accTy.getElementType().isF32()) ||
        (lhsTy.getElementType().isSignlessInteger(8) &&
         !accTy.getElementType().isSignlessInteger(32)))
      return rewriter.notifyMatchFailure(contractOp,
                                         "Only F32 for BF16 or Int32 for Int8 "
                                         "accumulation type is supported.");
 
    Operation *accReadOp =
        traceToVectorReadLikeParentOperation(contractOp.getAcc());
 
    // Only the contract result's first consumer is needed, not the final
    // store. This keeps the lowering independent of the epilogue ops (truncf,
    // bias add, ReLU, ...) that sit between the contraction and the write.
    Value resultChainEnd = contractionUsersAfterYield(contractOp.getResult());
 
    if (!accReadOp || !resultChainEnd)
      return rewriter.notifyMatchFailure(
          contractOp, "The ACC operand of the vector.contract should be a "
                      "transfer_read or a load. And, the result should have a "
                      "single-use chain to its consumer.");
 
    bool isAccZeroVectorConstant = isZeroVectorConstant(accReadOp);
 
    Block *resultBlock = resultChainEnd.user_begin()->getBlock();
 
    Type ipType = rewriter.getBF16Type();
    Type opType = rewriter.getF32Type();
 
    if (lhsTy.getElementType().isSignlessInteger(8)) {
      ipType = rewriter.getIntegerType(8);
      opType = rewriter.getIntegerType(32);
    }
 
    if (lhsTy.getElementType().isF8E4M3FN())
      ipType = rewriter.getF8E4M3FNType();
 
    if (lhsTy.getElementType().isF8E5M2())
      ipType = rewriter.getF8E5M2Type();
 
    if (accReadOp->getBlock() == contractOp->getBlock() &&
        resultBlock != contractOp->getBlock())
      return rewriter.notifyMatchFailure(
          contractOp, "The accumulator store is in different block.");
 
    if (accReadOp->getBlock() != contractOp->getBlock() &&
        resultBlock == contractOp->getBlock())
      return rewriter.notifyMatchFailure(
          contractOp, "The accumulator read is in different block.");
 
    if (!(isReadSrcMemref(contractOp.getLhs()) &&
          isReadSrcMemref(contractOp.getRhs())))
      return rewriter.notifyMatchFailure(
          contractOp, "The LHS or RHS src is not a MemRef type.");
 
    if (isVnni && !(isVnniOperandRead(contractOp.getLhs(), blockingFactor) &&
                    isVnniOperandRead(contractOp.getRhs(), blockingFactor)))
      return rewriter.notifyMatchFailure(
          contractOp, "The LHS or RHS src does not read whole VNNI groups.");
 
    unsigned int dimValue = blockingFactor;
    if (!isVnni)
      dimValue = 16 * blockingFactor;
 
    // Case 1: For just one VC rewrite. Where all accumulator read/write
    // within the same block.
    if (accReadOp->getBlock() == contractOp->getBlock() &&
        resultBlock == contractOp->getBlock()) {
 
      if (!isReadSrcMemref(contractOp.getAcc()))
        return rewriter.notifyMatchFailure(contractOp,
                                           "The ACC src is not a MemRef type.");
 
      bool collapse = false;
      if (isVnni)
        collapse = true;
 
      LogicalResult validate = validateContractOps(
          rewriter, contractOp, dimValue, Value(), Value(), false);
 
      if (failed(validate))
        return rewriter.notifyMatchFailure(
            contractOp, "The contract operation doesn't satisfy the operands "
                        "dimensions. M, N, and vnni dims are 16, 16, and 2/4. "
                        "The rest dims should be 1. Op should have one user.");
 
      Location loc = contractOp.getLoc();
 
      auto srcIndxLhs = getSrcIndxValue(rewriter, contractOp.getLoc(),
                                        contractOp.getLhs(), collapse);
      if (failed(srcIndxLhs))
        return rewriter.notifyMatchFailure(contractOp,
                                           "Failed to get the LHS src.");
      auto [srcBuffLhs, indicesLhs] = *srcIndxLhs;
 
      auto srcIndxRhs = getSrcIndxValue(rewriter, contractOp.getLoc(),
                                        contractOp.getRhs(), collapse);
      if (failed(srcIndxRhs))
        return rewriter.notifyMatchFailure(contractOp,
                                           "Failed to get the RHS src.");
      auto rhsSrc = *srcIndxRhs;
      auto srcBuffRhs = rhsSrc.first;
      auto indicesRhs = rhsSrc.second;
 
      auto srcIndxAcc = getSrcIndxValue(rewriter, contractOp.getLoc(),
                                        contractOp.getAcc(), false);
      if (failed(srcIndxAcc))
        return rewriter.notifyMatchFailure(contractOp,
                                           "Failed to get the ACC src.");
      auto [srcBuffAcc, indicesAcc] = *srcIndxAcc;
 
      Value c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
 
      // amx.tile_loads
      auto tileType = amx::TileType::get({16, (16 * blockingFactor)}, ipType);
      auto loadLhs = amx::TileLoadOp::create(rewriter, loc, tileType,
                                             srcBuffLhs, indicesLhs);
 
      // Create the subview and then load.
      amx::TileLoadOp loadRhs;
      if (!isVnni) {
        VectorType vecTy;
        SmallVector<OpFoldResult> indexVals;
        llvm::TypeSwitch<Operation *>(contractOp.getRhs().getDefiningOp())
            .Case<TransferReadOp, LoadOp>([&](auto readOp) {
              indexVals = SmallVector<OpFoldResult>(readOp.getIndices().begin(),
                                                    readOp.getIndices().end());
              vecTy = readOp.getType();
            });
        auto one = rewriter.getIndexAttr(1);
        SmallVector<OpFoldResult> strides(indexVals.size(), one);
        // A rank-reducing read drops leading unit dims. Append them back for
        // the subview.
        SmallVector<int64_t> subviewShape(indexVals.size() - vecTy.getRank(),
                                          1);
        llvm::append_range(subviewShape, vecTy.getShape());
        SmallVector<OpFoldResult> sizes =
            getAsIndexOpFoldResult(contractOp->getContext(), subviewShape);
        auto subview = memref::SubViewOp::create(rewriter, loc, srcBuffRhs,
                                                 indexVals, sizes, strides);
        auto bufferType = MemRefType::get({16, (16 * blockingFactor)}, ipType);
        auto packedBuffer = memref::AllocaOp::create(rewriter, loc, bufferType);
 
        // Create a loop that does online packing.
        Value step =
            arith::ConstantIndexOp::create(rewriter, loc, blockingFactor);
        Value uBound = arith::ConstantIndexOp::create(rewriter, loc,
                                                      (blockingFactor * 16));
        Value nextLoadIndx =
            arith::ConstantIndexOp::create(rewriter, loc, (blockingFactor / 2));
        Value nextStoreIndx = arith::ConstantIndexOp::create(
            rewriter, loc, 16 * (blockingFactor / 2));
 
        scf::ForOp::create(
            rewriter, loc, c0, uBound, step, ValueRange{},
            [&](OpBuilder &nestedBuilder, Location loc, Value iv,
                ValueRange iterArgs) {
              Value i1_load =
                  arith::AddIOp::create(rewriter, loc, nextLoadIndx, iv);
 
              // The source subview already accounts for the leading unit
              // dimensions.
              size_t numOffsets = indicesRhs.size();
              SmallVector<Value> indices(numOffsets, c0);
              indices[numOffsets - 2] = iv;
              auto vec1 = vector::LoadOp::create(
                  rewriter, loc,
                  VectorType::get(16 * (blockingFactor / 2), ipType), subview,
                  indices);
 
              indices[numOffsets - 2] = i1_load;
              auto vec2 = vector::LoadOp::create(
                  rewriter, loc,
                  VectorType::get(16 * (blockingFactor / 2), ipType), subview,
                  indices);
 
              vector::ShuffleOp writeVal1;
              vector::ShuffleOp writeVal2;
 
              if (blockingFactor == 2) {
 
                writeVal1 = vector::ShuffleOp::create(
                    rewriter, loc, VectorType::get({16}, ipType), vec1, vec2,
                    ArrayRef<int64_t>{0, 16, 1, 17, 2, 18, 3, 19, 4, 20, 5, 21,
                                      6, 22, 7, 23});
 
                writeVal2 = vector::ShuffleOp::create(
                    rewriter, loc, VectorType::get({16}, ipType), vec1, vec2,
                    ArrayRef<int64_t>{8, 24, 9, 25, 10, 26, 11, 27, 12, 28, 13,
                                      29, 14, 30, 15, 31});
              }
 
              if (blockingFactor == 4) {
                writeVal1 = vector::ShuffleOp::create(
                    rewriter, loc, VectorType::get({32}, ipType), vec1, vec2,
                    ArrayRef<int64_t>{0, 16, 32, 48, 1, 17, 33, 49,
                                      2, 18, 34, 50, 3, 19, 35, 51,
                                      4, 20, 36, 52, 5, 21, 37, 53,
                                      6, 22, 38, 54, 7, 23, 39, 55});
 
                writeVal2 = vector::ShuffleOp::create(
                    rewriter, loc, VectorType::get({32}, ipType), vec1, vec2,
                    ArrayRef<int64_t>{8,  24, 40, 56, 9,  25, 41, 57,
                                      10, 26, 42, 58, 11, 27, 43, 59,
                                      12, 28, 44, 60, 13, 29, 45, 61,
                                      14, 30, 46, 62, 15, 31, 47, 63});
              }
 
              auto rem = arith::DivUIOp::create(
                  rewriter, loc, rewriter.getIndexType(), iv, step);
 
              vector::StoreOp::create(rewriter, loc, writeVal1, packedBuffer,
                                      ValueRange{rem, c0});
              vector::StoreOp::create(rewriter, loc, writeVal2, packedBuffer,
                                      ValueRange{rem, nextStoreIndx});
 
              scf::YieldOp::create(nestedBuilder, loc);
            });
        loadRhs = amx::TileLoadOp::create(rewriter, loc, tileType, packedBuffer,
                                          ValueRange{c0, c0});
      } else {
 
        loadRhs = amx::TileLoadOp::create(rewriter, loc, tileType, srcBuffRhs,
                                          indicesRhs);
      }
 
      auto tileTypeAcc = amx::TileType::get({16, 16}, opType);
      auto loadAcc = amx::TileLoadOp::create(rewriter, loc, tileTypeAcc,
                                             srcBuffAcc, indicesAcc);
 
      // Tiled dot-product.
      Value dp;
      if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN())
        dp = amx::TileMulFOp::create(rewriter, loc, tileTypeAcc, loadLhs,
                                     loadRhs, loadAcc);
 
      if (ipType.isSignlessInteger(8))
        dp = amx::TileMulIOp::create(rewriter, loc, tileTypeAcc, loadLhs,
                                     loadRhs, loadAcc);
 
      auto bufferType = MemRefType::get({16, 16}, opType);
      auto resultBuffer = memref::AllocaOp::create(rewriter, loc, bufferType);
 
      amx::TileStoreOp::create(rewriter, loc, resultBuffer, ValueRange{c0, c0},
                               dp);
 
      auto vectorType = VectorType::get({16, 16}, opType);
      int64_t srcRank =
          (dyn_cast<ShapedType>(resultBuffer.getType())).getRank();
      Value padding = ub::PoisonOp::create(rewriter, loc, opType);
      auto map = AffineMap::getMinorIdentityMap(srcRank, vectorType.getRank(),
                                                rewriter.getContext());
      SmallVector<bool> inBounds(vectorType.getRank(), true);
 
      Value vecRow = vector::TransferReadOp::create(
          rewriter, loc, vectorType, resultBuffer, ValueRange{c0, c0}, padding,
          map, inBounds);
 
      Value resultOp = contractionUsersAfterYield(contractOp.getResult());
      if (auto vecType = dyn_cast<VectorType>(resultOp.getType()))
        vecRow = vector::ShapeCastOp::create(rewriter, loc, vecType, vecRow);
 
      rewriter.replaceAllUsesWith(resultOp, vecRow);
      return success();
    }
 
    // Case 2: The acc are passed as iter args through the reduction loop.
    // We support, reduction loop depth until 2. TODO: Support for n-depth
    // reduction loop.
    // TODOs: Re-factor 2a and 2b.
    SmallVector<scf::ForOp> loopLists;
    Operation *current = contractOp;
    while (true) {
      Operation *parent = current->getParentOfType<scf::ForOp>();
 
      if (!parent) {
        // The accumulator initialization can be hoisted above an enclosing
        // parallel region (scf.parallel/scf.forall) when the register tile
        // matches the problem size and the M/N register loops fold away. In
        // that case the reduction loop(s) collected so far are still valid to
        // rewrite, so stop climbing instead of bailing out.
        if (!loopLists.empty())
          break;
        return rewriter.notifyMatchFailure(
            contractOp,
            "Accumulator read and contract op not within scf.for op");
      }
 
      loopLists.push_back(dyn_cast<scf::ForOp>(parent));
 
      if (resultBlock == parent->getBlock()) {
        break;
      }
 
      current = parent;
    }
 
    if (loopLists.size() > 2 || loopLists.size() == 0)
      return rewriter.notifyMatchFailure(
          contractOp, "Rewrite is supported until reduction loop depth of 2.");
 
    auto srcIndxLhs = getSrcIndxValue(rewriter, contractOp.getLoc(),
                                      contractOp.getLhs(), false);
    if (failed(srcIndxLhs))
      return rewriter.notifyMatchFailure(contractOp,
                                         "Failed to get the LHS src.");
    auto [srcBuffLhs, indicesLhs] = *srcIndxLhs;
 
    auto srcIndxRhs = getSrcIndxValue(rewriter, contractOp.getLoc(),
                                      contractOp.getRhs(), false);
    if (failed(srcIndxRhs))
      return rewriter.notifyMatchFailure(contractOp,
                                         "Failed to get the RHS src.");
    auto [srcBuffRhs, indicesRhs] = *srcIndxRhs;
    Operation *vectorOpLhs;
    llvm::TypeSwitch<Operation *>(contractOp.getLhs().getDefiningOp())
        .Case<TransferReadOp, LoadOp>([&](auto readOp) {
          vectorOpLhs = readOp.getBase().getDefiningOp();
        });
 
    Operation *vectorOpRhs;
    llvm::TypeSwitch<Operation *>(contractOp.getRhs().getDefiningOp())
        .Case<TransferReadOp, LoadOp>([&](auto readOp) {
          vectorOpRhs = readOp.getBase().getDefiningOp();
        });
 
    if (!vectorOpLhs || !vectorOpRhs)
      return rewriter.notifyMatchFailure(
          contractOp, "Failed to find LHS or RHS read source operation");
 
    // Retrive all the contaction operation within the loop.
    SmallVector<vector::ContractionOp> ops;
    // Accumulators whose initial value passed into the rewritten reduction
    // loops is neither read from memory nor zero, e.g. an iter_arg of an
    // enclosing loop. Paired with the index of their contraction in `ops`.
    SmallVector<std::pair<size_t, Value>> carriedAccs;
    for (Operation &op : loopLists[0].getBody()->getOperations()) {
 
      if (auto contract = dyn_cast<vector::ContractionOp>(op)) {
        if (isVnni && !(isVnniOperandRead(contract.getLhs(), blockingFactor) &&
                        isVnniOperandRead(contract.getRhs(), blockingFactor)))
          return rewriter.notifyMatchFailure(
              contractOp, "The LHS or RHS src of an associated contract "
                          "operation does not read whole VNNI groups.");
 
        LogicalResult validate = validateContractOps(
            rewriter, contract, dimValue, srcBuffLhs, srcBuffRhs, true);
 
        if (failed(validate))
          return rewriter.notifyMatchFailure(
              contractOp,
              "The associated contract operations doesn't satisfy "
              "the re-write conditions either the dimensions are "
              "wrong or MemRef source are different or many users.");
 
        Operation *contractReadOp =
            traceToVectorReadLikeParentOperation(contract.getAcc());
 
        if (isAccZeroVectorConstant != isZeroVectorConstant(contractReadOp))
          return rewriter.notifyMatchFailure(
              contractOp, "The input acc should be a zero constant or "
                          "transfer_read for all contracts");
 
        Value accInit = getLoopNestInit(contract.getAcc(), loopLists);
        Operation *accInitOp = accInit.getDefiningOp();
        bool carried = !isa_and_present<TransferReadOp, LoadOp>(accInitOp) &&
                       !isZeroVectorConstant(accInitOp);
        if (carried) {
          if (loopLists.back().getRegion().isAncestor(
                  accInit.getParentRegion()))
            return rewriter.notifyMatchFailure(
                contractOp, "The accumulator should enter the reduction loops "
                            "through their init args.");
          carriedAccs.emplace_back(ops.size(), accInit);
        }
 
        ops.push_back(contract);
      }
    }
 
    // The accumulators read from memory are added back to all the results at
    // once, so they can't be mixed with carried ones.
    if (!isAccZeroVectorConstant && !carriedAccs.empty() &&
        carriedAccs.size() != ops.size())
      return rewriter.notifyMatchFailure(
          contractOp, "The accumulators read from memory can't be mixed with "
                      "carried in accumulators.");
 
    if (!isVnni) {
      unsigned int pairCount = 0;
      for (size_t j = 0; j < ops.size(); j++) {
        for (size_t i = j; i < ops.size(); i++) {
          if (i != j && validatePairVectorContract(ops[j], ops[i], true, 16))
            pairCount = pairCount + 2;
        }
      }
 
      if (pairCount != ops.size())
        return rewriter.notifyMatchFailure(
            contractOp, "Coudn't find the pair vector contract ");
    }
 
    scf::ForOp innerLoop;
    scf::ForOp outerLoop;
 
    // For VNNI layout, accumulator tiles stay in their original layout.
    // Load and store them back in place when possible.
    // Otherwise, start from zero tile and accumulate only the final result.
    std::optional<SmallVector<Operation *>> inPlaceAccReads, inPlaceAccWrites;
    if (isVnni)
      std::tie(inPlaceAccReads, inPlaceAccWrites) =
          getInPlaceAccAccesses(ops, /*writeOnly=*/isAccZeroVectorConstant);
    auto createInitTiles = [&](scf::ForOp loop) {
      if (inPlaceAccReads)
        return createAccTileLoads(rewriter, loop.getLoc(), opType, loop,
                                  *inPlaceAccReads);
      return createTileZeros(rewriter, loop.getLoc(), opType, loop, ops.size());
    };
 
    scf::ForOp newLoop;
    // Case 2a: Reduction loop depth is 2.
    if (loopLists.size() == 2) {
      outerLoop = loopLists[1];
      innerLoop = loopLists[0];
 
      LogicalResult validateOuterLoopStep =
          validateLoopStep(rewriter, outerLoop.getStep(), 1);
      if (failed(validateOuterLoopStep))
        return rewriter.notifyMatchFailure(contractOp, "Invalid loop step.");
 
      int64_t stepValue = 16;
      if (!isVnni)
        stepValue = stepValue * blockingFactor;
      LogicalResult validateInnerLoopStep =
          validateLoopStep(rewriter, innerLoop.getStep(), stepValue);
      if (failed(validateInnerLoopStep))
        return rewriter.notifyMatchFailure(
            contractOp, "Invalid loop step. The step should be 32 for BF16 and "
                        "64 for Int8/F8.");
 
      SmallVector<Value> loopItrArgs = createInitTiles(outerLoop);
      rewriter.setInsertionPoint(outerLoop);
 
      if (isVnni) {
        newLoop = scf::ForOp::create(
            rewriter, outerLoop.getLoc(), outerLoop.getLowerBound(),
            outerLoop.getUpperBound(), outerLoop.getStep(), loopItrArgs,
            [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop,
                Value ivOuterLoop, ValueRange iterArgsOuterLoop) {
              auto newInnerLoop = createLoops(
                  rewriter, innerLoop.getLoc(), innerLoop.getLowerBound(),
                  innerLoop.getUpperBound(), innerLoop.getStep(),
                  iterArgsOuterLoop, ipType, opType, blockingFactor, isVnni,
                  vectorOpLhs, vectorOpRhs, contractOp, outerLoop, innerLoop,
                  ops, ivOuterLoop, nullptr, true, nullptr, false, false);
 
              scf::YieldOp::create(rewriterOuterLoop, locOuterLoop,
                                   newInnerLoop.getResults());
            });
 
      } else {
 
        bool isInnerLoopUBLarger = false;
        bool isInnerLoopUBHasOddQuot = false;
 
        int64_t ubVal = 16 * blockingFactor;
        Value ub = innerLoop.getUpperBound();
        if (auto constOp = ub.getDefiningOp<arith::ConstantOp>()) {
          if (auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue())) {
            ubVal = intAttr.getInt();
          }
        }
 
        isInnerLoopUBLarger = ubVal > 16 * blockingFactor;
        isInnerLoopUBHasOddQuot =
            (((ubVal / (16 * blockingFactor)) % 2) == 1) && isInnerLoopUBLarger;
 
        rewriter.setInsertionPoint(outerLoop);
 
        auto c0 =
            arith::ConstantIndexOp::create(rewriter, outerLoop.getLoc(), 0);
        auto c1 =
            arith::ConstantIndexOp::create(rewriter, outerLoop.getLoc(), 1);
        auto spillLoopBound = arith::ConstantIndexOp::create(
            rewriter, outerLoop.getLoc(), 16 * blockingFactor);
 
        Value spillOuterLoop = arith::SubIOp::create(
            rewriter, outerLoop.getLoc(), outerLoop.getUpperBound(), c1);
        Value spillInnerLoop =
            arith::SubIOp::create(rewriter, innerLoop.getLoc(),
                                  innerLoop.getUpperBound(), spillLoopBound);
        auto bufferType =
            MemRefType::get({2, 32, (blockingFactor * 16)}, ipType);
        auto packedBuffer =
            memref::AllocaOp::create(rewriter, outerLoop.getLoc(), bufferType);
 
        // First Shuffling outside the reduction loops
        IRMapping rhsMapping;
        rhsMapping.map(
            vectorOpRhs->getOperand(
                getIndexPosition(contractOp.getRhs(), outerLoop) + 1),
            outerLoop.getLowerBound());
        rhsMapping.map(
            vectorOpRhs->getOperand(
                getIndexPosition(contractOp.getRhs(), innerLoop) + 1),
            innerLoop.getLowerBound());
        auto rhsClone = rewriter.clone(*vectorOpRhs, rhsMapping);
 
        Value quotient_batch = arith::DivUIOp::create(
            rewriter, outerLoop.getLoc(), outerLoop.getLowerBound(),
            outerLoop.getStep());
        Value quotient_k = arith::DivUIOp::create(rewriter, outerLoop.getLoc(),
                                                  innerLoop.getLowerBound(),
                                                  innerLoop.getStep());
 
        Value quotient_add = arith::AddIOp::create(rewriter, outerLoop.getLoc(),
                                                   quotient_batch, quotient_k);
        Value c2 =
            arith::ConstantIndexOp::create(rewriter, outerLoop.getLoc(), 2);
        Value rem = arith::RemUIOp::create(rewriter, outerLoop.getLoc(),
                                           quotient_add, c2);
 
        performShuffle(rewriter, outerLoop.getLoc(), rhsClone->getResult(0),
                       ipType, blockingFactor, packedBuffer, rem);
 
        // First Set of Loops
        auto newLoopNonSpill = scf::ForOp::create(
            rewriter, outerLoop.getLoc(), outerLoop.getLowerBound(),
            spillOuterLoop, outerLoop.getStep(), loopItrArgs,
            [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop,
                Value ivOuterLoop, ValueRange iterArgsOuterLoop) {
              auto newInnerLoop1 = createLoops(
                  rewriter, innerLoop.getLoc(), innerLoop.getLowerBound(),
                  spillInnerLoop, innerLoop.getStep(), iterArgsOuterLoop,
                  ipType, opType, blockingFactor, isVnni, vectorOpLhs,
                  vectorOpRhs, contractOp, outerLoop, innerLoop, ops,
                  ivOuterLoop, packedBuffer, true, spillLoopBound,
                  isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
 
              auto newInnerLoop = createLoops(
                  rewriter, innerLoop.getLoc(), spillInnerLoop,
                  innerLoop.getUpperBound(), innerLoop.getStep(),
                  newInnerLoop1.getResults(), ipType, opType, blockingFactor,
                  isVnni, vectorOpLhs, vectorOpRhs, contractOp, outerLoop,
                  innerLoop, ops, ivOuterLoop, packedBuffer, true, c0,
                  isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
 
              scf::YieldOp::create(rewriterOuterLoop, locOuterLoop,
                                   newInnerLoop.getResults());
            });
 
        // Last set of Loops
        newLoop = scf::ForOp::create(
            rewriter, outerLoop.getLoc(), spillOuterLoop,
            outerLoop.getUpperBound(), outerLoop.getStep(),
            newLoopNonSpill.getResults(),
            [&](OpBuilder &rewriterOuterLoop, Location locOuterLoop,
                Value ivOuterLoop, ValueRange iterArgsOuterLoop) {
              auto newInnerLoop1 = createLoops(
                  rewriter, innerLoop.getLoc(), innerLoop.getLowerBound(),
                  spillInnerLoop, innerLoop.getStep(), iterArgsOuterLoop,
                  ipType, opType, blockingFactor, isVnni, vectorOpLhs,
                  vectorOpRhs, contractOp, outerLoop, innerLoop, ops,
                  ivOuterLoop, packedBuffer, true, spillLoopBound,
                  isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
 
              auto newInnerLoop = createLoops(
                  rewriter, innerLoop.getLoc(), spillInnerLoop,
                  innerLoop.getUpperBound(), innerLoop.getStep(),
                  newInnerLoop1.getResults(), ipType, opType, blockingFactor,
                  isVnni, vectorOpLhs, vectorOpRhs, contractOp, outerLoop,
                  innerLoop, ops, ivOuterLoop, packedBuffer, false, c0,
                  isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
 
              scf::YieldOp::create(rewriterOuterLoop, locOuterLoop,
                                   newInnerLoop.getResults());
            });
      }
    }
 
    // Case 2b: Reduction loop depth is 1.
    if (loopLists.size() == 1) {
 
      innerLoop = loopLists[0];
      int64_t stepValue = 16;
      if (!isVnni)
        stepValue = stepValue * blockingFactor;
 
      LogicalResult validateInnerLoopStep =
          validateLoopStep(rewriter, innerLoop.getStep(), stepValue);
      if (failed(validateInnerLoopStep))
        return rewriter.notifyMatchFailure(
            contractOp,
            "Invalid loop step. The step should be 32 for BF16 and "
            "64 for Int8/F8 or 1 if it is rduction loop other than K.");
 
      SmallVector<Value> loopItrArgs = createInitTiles(innerLoop);
      rewriter.setInsertionPoint(innerLoop);
 
      if (isVnni) {
        newLoop = createLoops(
            rewriter, innerLoop.getLoc(), innerLoop.getLowerBound(),
            innerLoop.getUpperBound(), innerLoop.getStep(), loopItrArgs, ipType,
            opType, blockingFactor, isVnni, vectorOpLhs, vectorOpRhs,
            contractOp, nullptr, innerLoop, ops, nullptr, nullptr, true,
            nullptr, false, false);
 
      } else {
 
        bool isInnerLoopUBLarger = false;
        bool isInnerLoopUBHasOddQuot = false;
 
        int64_t ubVal = 16 * blockingFactor;
        Value ub = innerLoop.getUpperBound();
        if (auto constOp = ub.getDefiningOp<arith::ConstantOp>()) {
          if (auto intAttr = dyn_cast<IntegerAttr>(constOp.getValue())) {
            ubVal = intAttr.getInt();
          }
        }
 
        isInnerLoopUBLarger = ubVal > 16 * blockingFactor;
        isInnerLoopUBHasOddQuot =
            (((ubVal / (16 * blockingFactor)) % 2) == 1) && isInnerLoopUBLarger;
 
        rewriter.setInsertionPoint(innerLoop);
 
        auto c0 =
            arith::ConstantIndexOp::create(rewriter, innerLoop.getLoc(), 0);
        int64_t offset = 16 * blockingFactor;
        if (auto cst =
                innerLoop.getStep().getDefiningOp<arith::ConstantIndexOp>())
          offset = cst.value();
 
        auto spillLoopBound = arith::ConstantIndexOp::create(
            rewriter, innerLoop.getLoc(), offset);
        Value spillInnerLoop =
            arith::SubIOp::create(rewriter, innerLoop.getLoc(),
                                  innerLoop.getUpperBound(), spillLoopBound);
 
        auto bufferType =
            MemRefType::get({2, 32, (blockingFactor * 16)}, ipType);
        auto packedBuffer =
            memref::AllocaOp::create(rewriter, innerLoop.getLoc(), bufferType);
 
        // First Shuffling outside the reduction loops
        IRMapping rhsMapping;
        rhsMapping.map(
            vectorOpRhs->getOperand(
                getIndexPosition(contractOp.getRhs(), innerLoop) + 1),
            innerLoop.getLowerBound());
        auto rhsClone = rewriter.clone(*vectorOpRhs, rhsMapping);
 
        Value quotient_k = arith::DivUIOp::create(rewriter, innerLoop.getLoc(),
                                                  innerLoop.getLowerBound(),
                                                  innerLoop.getStep());
        Value c2 =
            arith::ConstantIndexOp::create(rewriter, innerLoop.getLoc(), 2);
        Value rem = arith::RemUIOp::create(rewriter, innerLoop.getLoc(),
                                           quotient_k, c2);
 
        performShuffle(rewriter, innerLoop.getLoc(), rhsClone->getResult(0),
                       ipType, blockingFactor, packedBuffer, rem);
 
        auto newLoopNonSpill = createLoops(
            rewriter, innerLoop.getLoc(), innerLoop.getLowerBound(),
            spillInnerLoop, innerLoop.getStep(), loopItrArgs, ipType, opType,
            blockingFactor, isVnni, vectorOpLhs, vectorOpRhs, contractOp,
            nullptr, innerLoop, ops, nullptr, packedBuffer, true,
            spillLoopBound, isInnerLoopUBLarger, isInnerLoopUBHasOddQuot);
 
        newLoop = createLoops(rewriter, innerLoop.getLoc(), spillInnerLoop,
                              innerLoop.getUpperBound(), innerLoop.getStep(),
                              newLoopNonSpill.getResults(), ipType, opType,
                              blockingFactor, isVnni, vectorOpLhs, vectorOpRhs,
                              contractOp, nullptr, innerLoop, ops, nullptr,
                              packedBuffer, false, c0, isInnerLoopUBLarger,
                              isInnerLoopUBHasOddQuot);
      }
 
      // This helps the final store back to the acc uses the same code for
      // the both reduction loop depth 1 or 2.
      outerLoop = innerLoop;
    }
 
    // Copy the amx tile accumulation results to a MemRef buffer, add the
    // initial accumulation value, and store back to the C-Matrix
    Location loc = outerLoop.getLoc();
 
    if (inPlaceAccWrites) {
      // The tiles already include the initial accumulator or were
      // zero-initialized: store each one where its contraction result was
      // written.
      for (auto [i, write] : llvm::enumerate(*inPlaceAccWrites)) {
        auto [base, indices] = *getAccTileAccess(write);
        rewriter.setInsertionPoint(write);
        amx::TileStoreOp::create(rewriter, write->getLoc(), base, indices,
                                 newLoop.getResult(i));
        rewriter.eraseOp(write);
      }
      return success();
    }
 
    // Carried-in accumulators are added to the results instead.
    bool addAccFromMemory = !isAccZeroVectorConstant && carriedAccs.empty();
 
    // Direct tile stores weren't possible, so we need a temporary buffer to
    // move tile register contents through memory.
    unsigned int M, N;
    Value srcBuffAcc;
    SmallVector<Value> indicesAcc;
 
    if (addAccFromMemory) {
      // We accumulate onto a register tile in memory, hence we let the
      // temporary buffer match its shape.
      llvm::TypeSwitch<Operation *>(accReadOp).Case<TransferReadOp, LoadOp>(
          [&](auto readOp) {
            srcBuffAcc = readOp.getOperand(0);
 
            auto indices = readOp.getIndices();
            indicesAcc.reserve(indices.size());
 
            llvm::transform(indices, std::back_inserter(indicesAcc),
                            [&](OpFoldResult ofr) {
                              return mlir::getValueOrCreateConstantIndexOp(
                                  rewriter, loc, ofr);
                            });
          });
      auto outputShapes = cast<MemRefType>(srcBuffAcc.getType()).getShape();
      M = outputShapes[outputShapes.size() - 2];
      N = outputShapes[outputShapes.size() - 1];
    } else {
      // Otherwise we can just pick a suitable shape to hold all result tiles.
      // For the VNNI-packed case, we pass the result from each tile register to
      // memory individually. For the online packing case, we ensure two tiles
      // are neighboring in the N-dimension to facilitate the shuffling.
      M = isVnni ? ops.size() * 16 : ops.size() / 2 * 16;
      N = isVnni ? 16 : 2 * 16;
    }
 
    SmallVector<Value> dps = newLoop.getResults();
    auto bufferType = MemRefType::get({M, N}, opType);
    auto resultBuffer = memref::AllocaOp::create(rewriter, loc, bufferType);
 
    // Store the amx tiled-dot product output into an MxN memref.
    for (unsigned int i = 0, k = 0; i < M; i = i + 16) {
      for (unsigned int j = 0; j < N; j = j + 16) {
        Value indexOp_i = arith::ConstantIndexOp::create(rewriter, loc, i);
        Value indexOp_j = arith::ConstantIndexOp::create(rewriter, loc, j);
        amx::TileStoreOp::create(rewriter, loc, resultBuffer,
                                 ValueRange{indexOp_i, indexOp_j}, dps[k]);
        k++;
      }
    }
 
    auto c0 = arith::ConstantIndexOp::create(rewriter, loc, 0);
    auto c16 = arith::ConstantIndexOp::create(rewriter, loc, 16);
    auto one = arith::ConstantIndexOp::create(rewriter, loc, 1);
    auto mBound = arith::ConstantIndexOp::create(rewriter, loc, M);
 
    if (!isVnni || addAccFromMemory) {
      // Create a loop that iterates over the MxN memref, retrieves two rows +
      // shuffles them, adds up the C element values and stores them to the temp
      // buffer.
      scf::ForOp::create(
          rewriter, loc, c0, mBound, one, ValueRange{},
          [&](OpBuilder &nestedBuilder, Location loc, Value iv,
              ValueRange iterArgs) {
            auto row = vector::LoadOp::create(rewriter, loc,
                                              VectorType::get(16, opType),
                                              resultBuffer, ValueRange{iv, c0});
 
            auto row2 = vector::LoadOp::create(
                rewriter, loc, VectorType::get(16, opType), resultBuffer,
                ValueRange{iv, c16});
 
            Value writeVal1 = row;
            Value writeVal2 = row2;
 
            if (!isVnni) {
              writeVal1 = vector::ShuffleOp::create(
                  rewriter, loc, VectorType::get(16, opType), row, row2,
                  ArrayRef<int64_t>{0, 1, 2, 3, 16, 17, 18, 19, 4, 5, 6, 7, 20,
                                    21, 22, 23});
 
              writeVal2 = vector::ShuffleOp::create(
                  rewriter, loc, VectorType::get(16, opType), row, row2,
                  ArrayRef<int64_t>{8, 9, 10, 11, 24, 25, 26, 27, 12, 13, 14,
                                    15, 28, 29, 30, 31});
            }
 
            if (addAccFromMemory) {
              indicesAcc[indicesAcc.size() - 2] = iv;
              indicesAcc[indicesAcc.size() - 1] = c0;
 
              Value valueCRow1 = vector::LoadOp::create(
                  rewriter, loc, VectorType::get(16, opType), srcBuffAcc,
                  indicesAcc);
              indicesAcc[indicesAcc.size() - 1] = c16;
 
              Value valueCRow2 = vector::LoadOp::create(
                  rewriter, loc, VectorType::get(16, opType), srcBuffAcc,
                  indicesAcc);
 
              if (ipType.isBF16() || ipType.isF8E5M2() || ipType.isF8E4M3FN()) {
                writeVal1 =
                    arith::AddFOp::create(rewriter, loc, writeVal1, valueCRow1);
 
                writeVal2 =
                    arith::AddFOp::create(rewriter, loc, writeVal2, valueCRow2);
              }
 
              if (ipType.isSignlessInteger(8)) {
                writeVal1 =
                    arith::AddIOp::create(rewriter, loc, writeVal1, valueCRow1);
 
                writeVal2 =
                    arith::AddIOp::create(rewriter, loc, writeVal2, valueCRow2);
              }
            }
 
            vector::StoreOp::create(rewriter, loc, writeVal1, resultBuffer,
                                    ValueRange{iv, c0});
            vector::StoreOp::create(rewriter, loc, writeVal2, resultBuffer,
                                    ValueRange{iv, c16});
 
            scf::YieldOp::create(nestedBuilder, loc);
          });
    }
 
    SmallVector<Value> writeResults;
    for (unsigned int i = 0; i < M; i = i + 16) {
      for (unsigned int j = 0; j < N; j = j + 16) {
        Value indexOp_i = arith::ConstantIndexOp::create(rewriter, loc, i);
        Value indexOp_j = arith::ConstantIndexOp::create(rewriter, loc, j);
 
        auto vectorType = VectorType::get({16, 16}, opType);
 
        int64_t srcRank =
            (dyn_cast<ShapedType>(resultBuffer.getType())).getRank();
        Value padding = ub::PoisonOp::create(rewriter, loc, opType);
        auto map = AffineMap::getMinorIdentityMap(srcRank, vectorType.getRank(),
                                                  rewriter.getContext());
        SmallVector<bool> inBounds(vectorType.getRank(), true);
 
        auto vec1 = vector::TransferReadOp::create(
            rewriter, loc, vectorType, resultBuffer,
            ValueRange{indexOp_i, indexOp_j}, padding, map, inBounds);
        writeResults.push_back(vec1);
      }
    }
 
    for (auto [idx, acc] : carriedAccs) {
      Type resultType = writeResults[idx].getType();
      if (acc.getType() != resultType)
        acc = vector::ShapeCastOp::create(rewriter, loc, resultType, acc);
      if (isa<FloatType>(opType))
        writeResults[idx] =
            arith::AddFOp::create(rewriter, loc, writeResults[idx], acc);
      else
        writeResults[idx] =
            arith::AddIOp::create(rewriter, loc, writeResults[idx], acc);
    }
 
    // Replace use of vector.contract with dot-products.
    for (size_t i = 0; i < ops.size(); i++) {
      vector::ContractionOp contOp = ops[i];
      Value vecRow = writeResults[i];
 
      Value resultWriteOp = contractionUsersAfterYield(contOp.getResult());
      if (auto vecType = dyn_cast<VectorType>(resultWriteOp.getType()))
        vecRow = vector::ShapeCastOp::create(rewriter, loc, vecType,
                                             writeResults[i]);
 
      rewriter.replaceAllUsesWith(resultWriteOp, vecRow);
    }
 
    return success();
  }
};
 
} // namespace
 
void x86::populateVectorContractToAMXDotProductPatterns(
    RewritePatternSet &patterns) {
  patterns.add<VectorContractToAMXDotProduct>(patterns.getContext());
}