BufferizationOps.cpp

Go to the documentation of this file.

//===----------------------------------------------------------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
 
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Bufferization/IR/BufferizableOpInterface.h"
#include "mlir/Dialect/Bufferization/IR/Bufferization.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/VerificationUtils.h"
#include "mlir/IR/Matchers.h"
#include "llvm/ADT/SmallVectorExtras.h"
#include <optional>
 
using namespace mlir;
using namespace mlir::bufferization;
 
//===----------------------------------------------------------------------===//
// Helper functions
//===----------------------------------------------------------------------===//
 
FailureOr<Value> mlir::bufferization::castOrReallocMemRefValue(
    OpBuilder &b, Value value, MemRefType destType,
    const BufferizationOptions &options) {
  auto srcType = llvm::cast<MemRefType>(value.getType());
 
  // Element type and rank must match.
  if (srcType.getElementType() != destType.getElementType())
    return failure();
  if (srcType.getRank() != destType.getRank())
    return failure();
 
  // In case the affine maps are different, we may need to use a copy if we go
  // from dynamic to static offset or stride (the canonicalization cannot know
  // at this point that it is really cast compatible).
  auto isGuaranteedCastCompatible = [](MemRefType source, MemRefType target) {
    int64_t sourceOffset, targetOffset;
    SmallVector<int64_t, 4> sourceStrides, targetStrides;
    if (failed(source.getStridesAndOffset(sourceStrides, sourceOffset)) ||
        failed(target.getStridesAndOffset(targetStrides, targetOffset)))
      return false;
    auto dynamicToStatic = [](int64_t a, int64_t b) {
      return ShapedType::isDynamic(a) && ShapedType::isStatic(b);
    };
    if (dynamicToStatic(sourceOffset, targetOffset))
      return false;
    for (auto it : zip(sourceStrides, targetStrides))
      if (dynamicToStatic(std::get<0>(it), std::get<1>(it)))
        return false;
    // A cast cannot safely zero out a non-zero offset. If the source has a
    // non-identity layout (e.g., a subview with a dynamic offset) and the
    // destination requires identity layout (zero offset, unit strides), a cast
    // would silently strip the offset and cause incorrect loads in the callee.
    // Fall through to the alloc+copy path instead.
    if (!source.getLayout().isIdentity() && target.getLayout().isIdentity())
      return false;
    return true;
  };
 
  // Note: If `areCastCompatible`, a cast is valid, but may fail at runtime. To
  // ensure that we only generate casts that always succeed at runtime, we check
  // a few extra conditions in `isGuaranteedCastCompatible`.
  if (memref::CastOp::areCastCompatible(srcType, destType) &&
      isGuaranteedCastCompatible(srcType, destType)) {
    Value casted = *options.castFn(b, value.getLoc(), destType, value);
    return casted;
  }
 
  auto loc = value.getLoc();
  SmallVector<Value, 4> dynamicOperands;
  for (int i = 0; i < destType.getRank(); ++i) {
    if (destType.getShape()[i] != ShapedType::kDynamic)
      continue;
    Value size = memref::DimOp::create(b, loc, value, i);
    dynamicOperands.push_back(size);
  }
 
  FailureOr<Value> copy = options.allocationFn(
      b, loc, destType, dynamicOperands, options.bufferAlignment);
  if (failed(copy))
    return failure();
  if (failed(options.memCpyFn(b, loc, value, *copy)))
    return failure();
  return copy;
}
 
/// Try to fold to_buffer(to_tensor(x)). If x's type and the result type of the
/// to_buffer op are different, a memref.cast is needed.
LogicalResult mlir::bufferization::foldToBufferToTensorPair(
    RewriterBase &rewriter, ToBufferOp toBuffer,
    const BufferizationOptions &options) {
  auto bufferToTensor = toBuffer.getTensor().getDefiningOp<ToTensorOp>();
  if (!bufferToTensor)
    return failure();
 
  Type srcType = bufferToTensor.getBuffer().getType();
  Type destType = toBuffer.getType();
 
  // Directly rewrite if the type did not change.
  if (srcType == destType) {
    rewriter.replaceOp(toBuffer, bufferToTensor.getBuffer());
    return success();
  }
 
  if (!llvm::isa<BaseMemRefType>(srcType) ||
      !llvm::isa<BaseMemRefType>(destType)) {
    // Non-builtin case: the best is to try the user-provided cast.
    auto replacement =
        options.castFn(rewriter, bufferToTensor.getBuffer().getLoc(), destType,
                       bufferToTensor.getBuffer());
    if (failed(replacement))
      return failure();
    rewriter.replaceOp(toBuffer, *replacement);
    return success();
  }
 
  auto rankedSrcType = llvm::dyn_cast<MemRefType>(srcType);
  auto rankedDestType = llvm::dyn_cast<MemRefType>(destType);
  auto unrankedSrcType = llvm::dyn_cast<UnrankedMemRefType>(srcType);
 
  // Ranked memref -> Ranked memref cast.
  if (rankedSrcType && rankedDestType) {
    FailureOr<Value> replacement = castOrReallocMemRefValue(
        rewriter, bufferToTensor.getBuffer(), rankedDestType, options);
    if (failed(replacement))
      return failure();
 
    rewriter.replaceOp(toBuffer, *replacement);
    return success();
  }
 
  // Unranked memref -> Ranked memref cast: May require a copy.
  // TODO: Not implemented at the moment.
  if (unrankedSrcType && rankedDestType)
    return failure();
 
  // Unranked/ranked memref -> unranked memref cast: No copy needed if the types
  // are cast-compatible.
  if (!memref::CastOp::areCastCompatible(srcType, destType))
    return failure();
 
  rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, destType,
                                              bufferToTensor.getBuffer());
  return success();
}
 
void mlir::bufferization::populateDynamicDimSizes(
    OpBuilder &b, Location loc, Value shapedValue,
    SmallVector<Value> &dynamicDims) {
  auto shapedType = llvm::cast<ShapedType>(shapedValue.getType());
  for (int64_t i = 0; i < shapedType.getRank(); ++i) {
    if (shapedType.isDynamicDim(i)) {
      if (llvm::isa<MemRefType>(shapedType)) {
        dynamicDims.push_back(memref::DimOp::create(b, loc, shapedValue, i));
      } else {
        assert(llvm::isa<RankedTensorType>(shapedType) && "expected tensor");
        dynamicDims.push_back(tensor::DimOp::create(b, loc, shapedValue, i));
      }
    }
  }
}
 
//===----------------------------------------------------------------------===//
// AllocTensorOp
//===----------------------------------------------------------------------===//
 
LogicalResult AllocTensorOp::verify() {
  auto type = dyn_cast<RankedTensorType>(getType());
  if (!type && (!getDynamicSizes().empty() || getSizeHint()))
    return emitError("takes dynamic sizes and a size hint only for a ranked "
                     "tensor result");
  if (getCopy() && !getDynamicSizes().empty())
    return emitError("dynamic sizes not needed when copying a tensor");
  if (type && !getCopy() &&
      failed(
          verifyDynamicDimensionCount(getOperation(), type, getDynamicSizes())))
    return failure();
  if (getCopy() && getCopy().getType() != getType())
    return emitError("expected that `copy` and return type match");
  return success();
}
 
void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
                          RankedTensorType type, ValueRange dynamicSizes) {
  build(builder, result, llvm::cast<TensorLikeType>(type), dynamicSizes,
        /*copy=*/Value(),
        /*size_hint=*/Value(),
        /*memory_space=*/IntegerAttr());
}
 
void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
                          RankedTensorType type, ValueRange dynamicSizes,
                          Value copy) {
  build(builder, result, llvm::cast<TensorLikeType>(type), dynamicSizes, copy);
}
 
void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
                          TensorType type, ValueRange dynamicSizes, Value copy,
                          IntegerAttr memorySpace) {
  build(builder, result, llvm::cast<TensorLikeType>(type), dynamicSizes, copy,
        /*size_hint=*/Value(), memorySpace);
}
 
void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
                          TensorLikeType type, ValueRange dynamicSizes,
                          Value copy) {
  build(builder, result, type, dynamicSizes, copy, /*size_hint=*/Value(),
        /*memory_space=*/Attribute());
}
 
void AllocTensorOp::build(OpBuilder &builder, OperationState &result,
                          TensorLikeType type, ValueRange dynamicSizes,
                          Value copy, Value sizeHint, Attribute memorySpace) {
  build(builder, result, Type(type), dynamicSizes, copy, sizeHint, memorySpace);
}
 
namespace {
/// Change the type of the result of a `bufferization.alloc_tensor` by making
/// the result type statically sized along dimension that in the original
/// operation where defined as dynamic, but the size was defined using a
/// `constant` op. For example:
///
///  %c5 = arith.constant 5: index
///  %0 = bufferization.alloc_tensor(%arg0, %c5) : tensor<?x?xf32>
///
///  to
///
///  %0 = bufferization.alloc_tensor(%arg0) : tensor<?x5xf32>
struct ReplaceStaticShapeDims : OpRewritePattern<AllocTensorOp> {
  using OpRewritePattern<AllocTensorOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(AllocTensorOp op,
                                PatternRewriter &rewriter) const override {
    // Dynamic sizes, and so this fold, belong to a ranked tensor result; any
    // other result carries none (verifier).
    auto type = dyn_cast<RankedTensorType>(op.getType());
    if (!type || op.getCopy())
      return failure();
    SmallVector<int64_t> newShape = llvm::to_vector(type.getShape());
    SmallVector<Value> newDynamicSizes;
    unsigned int dynValCounter = 0;
    for (int64_t i = 0; i < type.getRank(); ++i) {
      if (!type.isDynamicDim(i))
        continue;
      Value value = op.getDynamicSizes()[dynValCounter++];
      APInt intVal;
      if (matchPattern(value, m_ConstantInt(&intVal))) {
        int64_t dim = intVal.getSExtValue();
        if (dim >= 0)
          newShape[i] = intVal.getSExtValue();
        else
          newDynamicSizes.push_back(value);
      } else {
        newDynamicSizes.push_back(value);
      }
    }
    RankedTensorType newType = RankedTensorType::get(
        newShape, type.getElementType(), type.getEncoding());
    if (newType == type)
      return failure();
    auto newOp = AllocTensorOp::create(rewriter, op.getLoc(), newType,
                                       newDynamicSizes, /*copy=*/Value());
    rewriter.replaceOpWithNewOp<tensor::CastOp>(op, type, newOp);
    return success();
  }
};
 
struct FoldDimOfAllocTensorOp : public OpRewritePattern<tensor::DimOp> {
  using OpRewritePattern<tensor::DimOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(tensor::DimOp dimOp,
                                PatternRewriter &rewriter) const override {
    std::optional<int64_t> maybeConstantIndex = dimOp.getConstantIndex();
    auto allocTensorOp = dimOp.getSource().getDefiningOp<AllocTensorOp>();
    if (!allocTensorOp || !maybeConstantIndex)
      return failure();
    // Only a ranked tensor result has dynamic sizes to forward; any other
    // result carries none (verifier).
    auto type = dyn_cast<RankedTensorType>(allocTensorOp.getType());
    if (!type || *maybeConstantIndex < 0 ||
        *maybeConstantIndex >= type.getRank())
      return failure();
    if (!type.isDynamicDim(*maybeConstantIndex))
      return failure();
    rewriter.replaceOp(
        dimOp, allocTensorOp.getDynamicSize(rewriter, *maybeConstantIndex));
    return success();
  }
};
} // namespace
 
void AllocTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                                MLIRContext *ctx) {
  results.add<FoldDimOfAllocTensorOp, ReplaceStaticShapeDims>(ctx);
}
 
LogicalResult AllocTensorOp::reifyResultShapes(
    OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
  // The extents of a result that is not a ranked tensor are known only from
  // its buffer.
  auto type = dyn_cast<RankedTensorType>(getType());
  if (!type)
    return failure();
  auto shapes = llvm::map_to_vector<4>(
      llvm::seq<int64_t>(0, type.getRank()), [&](int64_t dim) -> OpFoldResult {
        if (type.isDynamicDim(dim))
          return getDynamicSize(builder, dim);
        return builder.getIndexAttr(type.getDimSize(dim));
      });
  reifiedReturnShapes.emplace_back(std::move(shapes));
  return success();
}
 
ParseResult AllocTensorOp::parse(OpAsmParser &parser, OperationState &result) {
  SmallVector<OpAsmParser::UnresolvedOperand> dynamicSizesOperands;
  if (parser.parseLParen() || parser.parseOperandList(dynamicSizesOperands) ||
      parser.parseRParen())
    return failure();
  ParseResult copyKeyword = parser.parseOptionalKeyword("copy");
  OpAsmParser::UnresolvedOperand copyOperand;
  if (copyKeyword.succeeded())
    if (parser.parseLParen() || parser.parseOperand(copyOperand) ||
        parser.parseRParen())
      return failure();
  ParseResult sizeHintKeyword = parser.parseOptionalKeyword("size_hint");
  OpAsmParser::UnresolvedOperand sizeHintOperand;
  if (sizeHintKeyword.succeeded())
    if (parser.parseEqual() || parser.parseOperand(sizeHintOperand))
      return failure();
 
  Attribute parsedProperties;
  if (AllocTensorOp::genericParseProperties(parser, parsedProperties))
    return failure();
  auto propertyDictionary = dyn_cast_or_null<DictionaryAttr>(parsedProperties);
  if (parsedProperties && !propertyDictionary)
    return parser.emitError(parser.getNameLoc(),
                            "expected properties dictionary");
 
  auto attrsLoc = parser.getCurrentLocation();
  if (parser.parseOptionalAttrDict(result.attributes))
    return failure();
  for (StringRef attrName : AllocTensorOp::getAttributeNames()) {
    if (result.attributes.get(attrName))
      return parser.emitError(attrsLoc)
             << "inherent attribute '" << attrName
             << "' cannot be parsed from attr-dict when strict properties in "
                "assembly format is enabled";
  }
  if (parser.parseColon())
    return failure();
 
  TensorLikeType type;
  if (parser.parseCustomTypeWithFallback(type))
    return failure();
  result.addTypes(type);
 
  Type indexType = parser.getBuilder().getIndexType();
  if (parser.resolveOperands(dynamicSizesOperands, indexType, result.operands))
    return failure();
  if (copyKeyword.succeeded())
    if (parser.resolveOperand(copyOperand, type, result.operands))
      return failure();
  if (sizeHintKeyword.succeeded())
    if (parser.resolveOperand(sizeHintOperand, indexType, result.operands))
      return failure();
  Builder &builder = parser.getBuilder();
  NamedAttrList properties(propertyDictionary ? propertyDictionary
                                              : builder.getDictionaryAttr({}));
  properties.set(AllocTensorOp::getOperandSegmentSizeAttr(),
                 builder.getDenseI32ArrayAttr(
                     {static_cast<int32_t>(dynamicSizesOperands.size()),
                      static_cast<int32_t>(copyKeyword.succeeded()),
                      static_cast<int32_t>(sizeHintKeyword.succeeded())}));
  propertyDictionary = properties.getDictionary(builder.getContext());
  auto emitError = [&]() {
    return mlir::emitError(result.location, "invalid properties ")
           << propertyDictionary << " for op " << result.name.getStringRef()
           << ": ";
  };
  if (failed(AllocTensorOp::setPropertiesFromParsedAttr(
          result.getOrAddProperties<Properties>(), propertyDictionary,
          emitError)))
    return failure();
  return success();
}
 
void AllocTensorOp::print(OpAsmPrinter &p) {
  p << "(" << getDynamicSizes() << ")";
  if (getCopy())
    p << " copy(" << getCopy() << ")";
  if (getSizeHint())
    p << " size_hint=" << getSizeHint();
  AllocTensorOp::printProperties(getContext(), p, getProperties(),
                                 /*elidedProps=*/getOperandSegmentSizeAttr());
  p.printOptionalAttrDict((*this)->getDiscardableAttrDictionary().getValue());
  p << " : ";
  auto type = getResult().getType();
  if (auto validType = llvm::dyn_cast<::mlir::TensorType>(type))
    p.printStrippedAttrOrType(validType);
  else
    p << type;
}
 
Value AllocTensorOp::getDynamicSize(OpBuilder &b, unsigned idx) {
  assert(isDynamicDim(idx) && "expected dynamic dim");
  if (getCopy())
    return tensor::DimOp::create(b, getLoc(), getCopy(), idx);
  return getOperand(getIndexOfDynamicSize(idx));
}
 
//===----------------------------------------------------------------------===//
// CloneOp
//===----------------------------------------------------------------------===//
 
OpFoldResult CloneOp::fold(FoldAdaptor adaptor) {
  return succeeded(memref::foldMemRefCast(*this)) ? getResult() : Value();
}
 
namespace {
 
/// Merge the clone and its source (by converting the clone to a cast) when
/// possible.
struct SimplifyClones : public OpRewritePattern<CloneOp> {
  using OpRewritePattern<CloneOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(CloneOp cloneOp,
                                PatternRewriter &rewriter) const override {
    if (cloneOp.use_empty()) {
      rewriter.eraseOp(cloneOp);
      return success();
    }
 
    Value source = cloneOp.getInput();
    if (source.getType() != cloneOp.getType() &&
        !memref::CastOp::areCastCompatible({source.getType()},
                                           {cloneOp.getType()}))
      return failure();
 
    // Aims to find the dealloc op for the canonical source
    // which otherwise could prevent removal of unnecessary allocs.
    Value canonicalSource = source;
    while (auto iface = dyn_cast_or_null<ViewLikeOpInterface>(
               canonicalSource.getDefiningOp())) {
      if (canonicalSource != iface.getViewDest()) {
        break;
      }
      canonicalSource = iface.getViewSource();
    }
 
    std::optional<Operation *> maybeCloneDeallocOp =
        memref::findDealloc(cloneOp.getOutput());
    // Skip if either of them has > 1 deallocate operations.
    if (!maybeCloneDeallocOp.has_value())
      return failure();
    std::optional<Operation *> maybeSourceDeallocOp =
        memref::findDealloc(canonicalSource);
    if (!maybeSourceDeallocOp.has_value())
      return failure();
    Operation *cloneDeallocOp = *maybeCloneDeallocOp;
    Operation *sourceDeallocOp = *maybeSourceDeallocOp;
 
    // If both are deallocated in the same block, their in-block lifetimes
    // might not fully overlap, so we cannot decide which one to drop.
    if (cloneDeallocOp && sourceDeallocOp &&
        cloneDeallocOp->getBlock() == sourceDeallocOp->getBlock())
      return failure();
 
    Block *currentBlock = cloneOp->getBlock();
    Operation *redundantDealloc = nullptr;
    if (cloneDeallocOp && cloneDeallocOp->getBlock() == currentBlock) {
      redundantDealloc = cloneDeallocOp;
    } else if (sourceDeallocOp && sourceDeallocOp->getBlock() == currentBlock) {
      redundantDealloc = sourceDeallocOp;
    }
 
    if (!redundantDealloc)
      return failure();
 
    // Safety check that there are no other deallocations inbetween
    // cloneOp and redundantDealloc, as otherwise we might deallocate an alias
    // of source before the uses of the clone. With alias information, we could
    // restrict this to only fail of the dealloc's operand is an alias
    // of the source.
    for (Operation *pos = cloneOp->getNextNode(); pos != redundantDealloc;
         pos = pos->getNextNode()) {
      // Bail if we run out of operations while looking for a deallocation op.
      if (!pos)
        return failure();
      auto effectInterface = dyn_cast<MemoryEffectOpInterface>(pos);
      if (!effectInterface)
        continue;
      if (effectInterface.hasEffect<MemoryEffects::Free>())
        return failure();
    }
 
    if (source.getType() != cloneOp.getType())
      source = memref::CastOp::create(rewriter, cloneOp.getLoc(),
                                      cloneOp.getType(), source);
    rewriter.replaceOp(cloneOp, source);
    rewriter.eraseOp(redundantDealloc);
    return success();
  }
};
 
} // namespace
 
void CloneOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                          MLIRContext *context) {
  results.add<SimplifyClones>(context);
}
 
//===----------------------------------------------------------------------===//
// MaterializeInDestinationOp
//===----------------------------------------------------------------------===//
 
LogicalResult MaterializeInDestinationOp::reifyResultShapes(
    OpBuilder &builder, ReifiedRankedShapedTypeDims &reifiedReturnShapes) {
  if (getOperation()->getNumResults() == 1) {
    assert(isa<TensorType>(getDest().getType()) && "expected tensor type");
    reifiedReturnShapes.resize(1,
                               SmallVector<OpFoldResult>(getType().getRank()));
    reifiedReturnShapes[0] =
        tensor::getMixedSizes(builder, getLoc(), getDest());
  }
  return success();
}
 
Value MaterializeInDestinationOp::buildSubsetExtraction(OpBuilder &builder,
                                                        Location loc) {
  if (isa<TensorType>(getDest().getType())) {
    // The subset is the entire destination tensor.
    return getDest();
  }
 
  // The "restrict" attribute is transferred from this op to the newly created
  // to_tensor op. If this op does not the "restrict" attribute, the subset
  // extraction cannot be built because there is no guarantee that there is no
  // pre-existing "restrict" to_tensor op with the same/an aliasing destination.
  if (!getRestrict())
    return {};
 
  // Build a bufferization.to_tensor op.
  assert(isa<BaseMemRefType>(getDest().getType()) && "expected memref type");
  assert(getRestrict() &&
         "expected that ops with memrefs dest have 'restrict'");
  setRestrict(false);
  return ToTensorOp::create(
      builder, loc, memref::getTensorTypeFromMemRefType(getDest().getType()),
      getDest(),
      /*restrict=*/true, getWritable());
}
 
bool MaterializeInDestinationOp::isEquivalentSubset(
    Value candidate, function_ref<bool(Value, Value)> equivalenceFn) {
  return equivalenceFn(getDest(), candidate);
}
 
SmallVector<Value>
MaterializeInDestinationOp::getValuesNeededToBuildSubsetExtraction() {
  return {getDest()};
}
 
OpOperand &MaterializeInDestinationOp::getSourceOperand() {
  return getOperation()->getOpOperand(0) /*source*/;
}
 
bool MaterializeInDestinationOp::operatesOnEquivalentSubset(
    SubsetOpInterface subsetOp,
    function_ref<bool(Value, Value)> equivalenceFn) {
  return false;
}
 
bool MaterializeInDestinationOp::operatesOnDisjointSubset(
    SubsetOpInterface subsetOp,
    function_ref<bool(Value, Value)> equivalenceFn) {
  return false;
}
 
LogicalResult MaterializeInDestinationOp::verify() {
  if (!isa<TensorType, BaseMemRefType>(getDest().getType()))
    return emitOpError("'dest' must be a tensor or a memref");
  if (auto destType = dyn_cast<TensorType>(getDest().getType())) {
    if (getOperation()->getNumResults() != 1)
      return emitOpError("tensor 'dest' implies exactly one tensor result");
    if (destType != getResult().getType())
      return emitOpError("result and 'dest' types must match");
  }
  if (isa<BaseMemRefType>(getDest().getType()) &&
      getOperation()->getNumResults() != 0)
    return emitOpError("memref 'dest' implies zero results");
  if (getRestrict() && !isa<BaseMemRefType>(getDest().getType()))
    return emitOpError("'restrict' is valid only for memref destinations");
  if (getWritable() != isa<BaseMemRefType>(getDest().getType()))
    return emitOpError("'writable' must be specified if and only if the "
                       "destination is of memref type");
  TensorType srcType = getSource().getType();
  ShapedType destType = cast<ShapedType>(getDest().getType());
  if (srcType.hasRank() != destType.hasRank())
    return emitOpError("source/destination shapes are incompatible");
  if (srcType.hasRank()) {
    if (failed(verifyRanksMatch(getOperation(), srcType, destType, "source",
                                "destination")))
      return failure();
    for (auto [src, dest] :
         llvm::zip(srcType.getShape(), destType.getShape())) {
      if (src == ShapedType::kDynamic || dest == ShapedType::kDynamic) {
        // Cannot verify dynamic dimension size. Assume that that they match at
        // runtime.
        continue;
      }
      if (src != dest)
        return emitOpError("source/destination shapes are incompatible");
    }
  }
  return success();
}
 
void MaterializeInDestinationOp::build(OpBuilder &builder,
                                       OperationState &state, Value source,
                                       Value dest) {
  auto destTensorType = dyn_cast<TensorType>(dest.getType());
  build(builder, state, /*result=*/destTensorType ? destTensorType : Type(),
        source, dest);
}
 
MutableOperandRange MaterializeInDestinationOp::getDpsInitsMutable() {
  return getDestMutable();
}
 
void MaterializeInDestinationOp::getEffects(
    SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
        &effects) {
  if (isa<BaseMemRefType>(getDest().getType()))
    effects.emplace_back(MemoryEffects::Write::get(), &getDestMutable(),
                         SideEffects::DefaultResource::get());
}
 
//===----------------------------------------------------------------------===//
// ToTensorOp
//===----------------------------------------------------------------------===//
 
OpFoldResult ToTensorOp::fold(FoldAdaptor) {
  if (auto toBuffer = getBuffer().getDefiningOp<ToBufferOp>())
    // Approximate alias analysis by conservatively folding only when no there
    // is no interleaved operation.
    if (toBuffer->getBlock() == this->getOperation()->getBlock() &&
        toBuffer->getNextNode() == this->getOperation())
      return toBuffer.getTensor();
  return {};
}
 
namespace {
struct DimOfToTensorFolder : public OpRewritePattern<tensor::DimOp> {
  using OpRewritePattern<tensor::DimOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(tensor::DimOp dimOp,
                                PatternRewriter &rewriter) const override {
    auto memrefToTensorOp = dimOp.getSource().getDefiningOp<ToTensorOp>();
    if (!memrefToTensorOp)
      return failure();
 
    rewriter.replaceOpWithNewOp<memref::DimOp>(
        dimOp, memrefToTensorOp.getBuffer(), dimOp.getIndex());
    return success();
  }
};
} // namespace
 
void ToTensorOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
  results.add<DimOfToTensorFolder>(context);
}
 
//===----------------------------------------------------------------------===//
// ToBufferOp
//===----------------------------------------------------------------------===//
 
OpFoldResult ToBufferOp::fold(FoldAdaptor) {
  if (auto memrefToTensor = getTensor().getDefiningOp<ToTensorOp>())
    if (memrefToTensor.getBuffer().getType() == getType())
      return memrefToTensor.getBuffer();
  return {};
}
 
namespace {
 
/// Replace tensor.cast + to_buffer by to_buffer + memref.cast.
struct ToBufferOfCast : public OpRewritePattern<ToBufferOp> {
  using OpRewritePattern<ToBufferOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(ToBufferOp toBuffer,
                                PatternRewriter &rewriter) const final {
    auto tensorCastOperand =
        toBuffer.getOperand().getDefiningOp<tensor::CastOp>();
    if (!tensorCastOperand)
      return failure();
    auto srcTensorType = llvm::dyn_cast<RankedTensorType>(
        tensorCastOperand.getOperand().getType());
    if (!srcTensorType)
      return failure();
    auto currentOutputMemRefType =
        dyn_cast<BaseMemRefType>(toBuffer.getResult().getType());
    if (!currentOutputMemRefType)
      return failure();
 
    auto memrefType = currentOutputMemRefType.cloneWith(
        srcTensorType.getShape(), srcTensorType.getElementType());
    Value memref = ToBufferOp::create(rewriter, toBuffer.getLoc(), memrefType,
                                      tensorCastOperand.getOperand(),
                                      toBuffer.getReadOnly());
    rewriter.replaceOpWithNewOp<memref::CastOp>(toBuffer, toBuffer.getType(),
                                                memref);
    return success();
  }
};
 
/// Canonicalize bufferization.to_tensor + bufferization.to_buffer. Insert a
/// cast if necessary.
struct ToBufferToTensorFolding : public OpRewritePattern<ToBufferOp> {
  using OpRewritePattern<ToBufferOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(ToBufferOp toBuffer,
                                PatternRewriter &rewriter) const final {
    BufferizationOptions options;
    options.bufferAlignment = 0;
    return foldToBufferToTensorPair(rewriter, toBuffer, options);
  }
};
 
/// Fold a load on a to_buffer operation into an tensor.extract on the
/// corresponding tensor.
struct LoadOfToBuffer : public OpRewritePattern<memref::LoadOp> {
  using OpRewritePattern<memref::LoadOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(memref::LoadOp load,
                                PatternRewriter &rewriter) const override {
    auto toBuffer = load.getMemref().getDefiningOp<ToBufferOp>();
    if (!toBuffer || !toBuffer.getReadOnly())
      return failure();
 
    rewriter.replaceOpWithNewOp<tensor::ExtractOp>(load, toBuffer.getTensor(),
                                                   load.getIndices());
    return success();
  }
};
 
/// Fold dim of a to_buffer into the dim of the tensor.
struct DimOfCastOp : public OpRewritePattern<memref::DimOp> {
  using OpRewritePattern<memref::DimOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(memref::DimOp dimOp,
                                PatternRewriter &rewriter) const override {
    auto castOp = dimOp.getSource().getDefiningOp<ToBufferOp>();
    if (!castOp)
      return failure();
    Value newSource = castOp.getOperand();
    rewriter.replaceOpWithNewOp<tensor::DimOp>(dimOp, newSource,
                                               dimOp.getIndex());
    return success();
  }
};
 
} // namespace
 
void ToBufferOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                             MLIRContext *context) {
  results.add<DimOfCastOp, LoadOfToBuffer, ToBufferOfCast,
              ToBufferToTensorFolding>(context);
}
 
std::optional<Operation *> CloneOp::buildDealloc(OpBuilder &builder,
                                                 Value alloc) {
  return memref::DeallocOp::create(builder, alloc.getLoc(), alloc)
      .getOperation();
}
 
std::optional<Value> CloneOp::buildClone(OpBuilder &builder, Value alloc) {
  return CloneOp::create(builder, alloc.getLoc(), alloc).getResult();
}
 
//===----------------------------------------------------------------------===//
// DeallocOp
//===----------------------------------------------------------------------===//
 
LogicalResult DeallocOp::inferReturnTypes(
    MLIRContext *context, std::optional<::mlir::Location> location,
    ValueRange operands, DictionaryAttr attributes, PropertyRef properties,
    RegionRange regions, SmallVectorImpl<Type> &inferredReturnTypes) {
  DeallocOpAdaptor adaptor(operands, attributes, properties, regions);
  inferredReturnTypes = SmallVector<Type>(adaptor.getRetained().size(),
                                          IntegerType::get(context, 1));
  return success();
}
 
LogicalResult DeallocOp::verify() {
  if (getMemrefs().size() != getConditions().size())
    return emitOpError(
        "must have the same number of conditions as memrefs to deallocate");
  if (getRetained().size() != getUpdatedConditions().size())
    return emitOpError("must have the same number of updated conditions "
                       "(results) as retained operands");
  return success();
}
 
static LogicalResult updateDeallocIfChanged(DeallocOp deallocOp,
                                            ValueRange memrefs,
                                            ValueRange conditions,
                                            PatternRewriter &rewriter) {
  if (deallocOp.getMemrefs() == memrefs &&
      deallocOp.getConditions() == conditions)
    return failure();
 
  rewriter.modifyOpInPlace(deallocOp, [&]() {
    deallocOp.getMemrefsMutable().assign(memrefs);
    deallocOp.getConditionsMutable().assign(conditions);
  });
  return success();
}
 
namespace {
 
/// Remove duplicate values in the list of memrefs to be deallocated. We need to
/// make sure the corresponding condition value is updated accordingly since
/// their two conditions might not cover the same set of cases. In that case, we
/// have to combine them (by computing the disjunction of them).
/// Example:
/// ```mlir
/// bufferization.dealloc (%arg0, %arg0 : ...) if (%arg1, %arg2)
/// ```
/// is canonicalized to
/// ```mlir
/// %0 = arith.ori %arg1, %arg2 : i1
/// bufferization.dealloc (%arg0 : memref<2xi32>) if (%0)
/// ```
struct DeallocRemoveDuplicateDeallocMemrefs
    : public OpRewritePattern<DeallocOp> {
  using OpRewritePattern<DeallocOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                PatternRewriter &rewriter) const override {
    // Unique memrefs to be deallocated.
    DenseMap<Value, unsigned> memrefToCondition;
    SmallVector<Value> newMemrefs, newConditions;
    for (auto [i, memref, cond] :
         llvm::enumerate(deallocOp.getMemrefs(), deallocOp.getConditions())) {
      if (memrefToCondition.count(memref)) {
        // If the dealloc conditions don't match, we need to make sure that the
        // dealloc happens on the union of cases.
        Value &newCond = newConditions[memrefToCondition[memref]];
        if (newCond != cond)
          newCond =
              arith::OrIOp::create(rewriter, deallocOp.getLoc(), newCond, cond);
      } else {
        memrefToCondition.insert({memref, newConditions.size()});
        newMemrefs.push_back(memref);
        newConditions.push_back(cond);
      }
    }
 
    // Return failure if we don't change anything such that we don't run into an
    // infinite loop of pattern applications.
    return updateDeallocIfChanged(deallocOp, newMemrefs, newConditions,
                                  rewriter);
  }
};
 
/// Remove duplicate values in the list of retained memrefs. We need to make
/// sure the corresponding result condition value is replaced properly.
/// Example:
/// ```mlir
/// %0:2 = bufferization.dealloc retain (%arg3, %arg3 : ...)
/// ```
/// is canonicalized to
/// ```mlir
/// %0 = bufferization.dealloc retain (%arg3 : memref<2xi32>)
/// ```
struct DeallocRemoveDuplicateRetainedMemrefs
    : public OpRewritePattern<DeallocOp> {
  using OpRewritePattern<DeallocOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                PatternRewriter &rewriter) const override {
    // Unique retained values
    DenseMap<Value, unsigned> seen;
    SmallVector<Value> newRetained;
    SmallVector<unsigned> resultReplacementIdx;
    unsigned i = 0;
    for (auto retained : deallocOp.getRetained()) {
      if (seen.count(retained)) {
        resultReplacementIdx.push_back(seen[retained]);
        continue;
      }
 
      seen[retained] = i;
      newRetained.push_back(retained);
      resultReplacementIdx.push_back(i++);
    }
 
    // Return failure if we don't change anything such that we don't run into an
    // infinite loop of pattern applications.
    if (newRetained.size() == deallocOp.getRetained().size())
      return failure();
 
    // We need to create a new op because the number of results is always the
    // same as the number of condition operands.
    auto newDeallocOp =
        DeallocOp::create(rewriter, deallocOp.getLoc(), deallocOp.getMemrefs(),
                          deallocOp.getConditions(), newRetained);
    SmallVector<Value> replacements(
        llvm::map_range(resultReplacementIdx, [&](unsigned idx) {
          return newDeallocOp.getUpdatedConditions()[idx];
        }));
    rewriter.replaceOp(deallocOp, replacements);
    return success();
  }
};
 
/// Erase deallocation operations where the variadic list of memrefs to
/// deallocate is empty. Example:
/// ```mlir
/// %0 = bufferization.dealloc retain (%arg0: memref<2xi32>)
/// ```
struct EraseEmptyDealloc : public OpRewritePattern<DeallocOp> {
  using OpRewritePattern<DeallocOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                PatternRewriter &rewriter) const override {
    if (deallocOp.getMemrefs().empty()) {
      Value constFalse = arith::ConstantOp::create(rewriter, deallocOp.getLoc(),
                                                   rewriter.getBoolAttr(false));
      rewriter.replaceOp(
          deallocOp, SmallVector<Value>(deallocOp.getUpdatedConditions().size(),
                                        constFalse));
      return success();
    }
    return failure();
  }
};
 
/// Removes memrefs from the deallocation list if their associated condition is
/// always 'false'.
///
/// Example:
/// ```
/// bufferization.dealloc (%arg0, %arg1 : memref<2xi32>, memref<2xi32>)
///                           if (%arg2, %false)
/// ```
/// becomes
/// ```
/// bufferization.dealloc (%arg0 : memref<2xi32>) if (%arg2)
/// ```
struct EraseAlwaysFalseDealloc : public OpRewritePattern<DeallocOp> {
  using OpRewritePattern<DeallocOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                PatternRewriter &rewriter) const override {
    SmallVector<Value> newMemrefs, newConditions;
    for (auto [memref, cond] :
         llvm::zip(deallocOp.getMemrefs(), deallocOp.getConditions())) {
      if (!matchPattern(cond, m_Zero())) {
        newMemrefs.push_back(memref);
        newConditions.push_back(cond);
      }
    }
 
    return updateDeallocIfChanged(deallocOp, newMemrefs, newConditions,
                                  rewriter);
  }
};
 
/// The `memref.extract_strided_metadata` is often inserted to get the base
/// memref if the operand is not already guaranteed to be the result of a memref
/// allocation operation. This canonicalization pattern removes this extraction
/// operation if the operand is now produced by an allocation operation (e.g.,
/// due to other canonicalizations simplifying the IR).
///
/// Example:
/// ```mlir
/// %alloc = memref.alloc() : memref<2xi32>
/// %base_memref, %offset, %size, %stride = memref.extract_strided_metadata
///   %alloc : memref<2xi32> -> memref<i32>, index, index, index
/// bufferization.dealloc (%base_memref : memref<i32>) if (%cond)
/// ```
/// is canonicalized to
/// ```mlir
/// %alloc = memref.alloc() : memref<2xi32>
/// bufferization.dealloc (%alloc : memref<2xi32>) if (%cond)
/// ```
struct SkipExtractMetadataOfAlloc : public OpRewritePattern<DeallocOp> {
  using OpRewritePattern<DeallocOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                PatternRewriter &rewriter) const override {
    SmallVector<Value> newMemrefs(
        llvm::map_range(deallocOp.getMemrefs(), [&](Value memref) {
          auto extractStridedOp =
              memref.getDefiningOp<memref::ExtractStridedMetadataOp>();
          if (!extractStridedOp)
            return memref;
          Value allocMemref = extractStridedOp.getOperand();
          auto allocOp = allocMemref.getDefiningOp<MemoryEffectOpInterface>();
          if (!allocOp)
            return memref;
          if (allocOp.getEffectOnValue<MemoryEffects::Allocate>(allocMemref))
            return allocMemref;
          return memref;
        }));
 
    return updateDeallocIfChanged(deallocOp, newMemrefs,
                                  deallocOp.getConditions(), rewriter);
  }
};
 
/// Removes pairs of `bufferization.dealloc` and alloc operations if there is no
/// other user of the allocated value and the allocating operation can be safely
/// removed. If the same value is present multiple times, this pattern relies on
/// other canonicalization patterns to remove the duplicate first.
///
/// Example:
/// ```mlir
/// %alloc = memref.alloc() : memref<2xi32>
/// bufferization.dealloc (%alloc, %arg0, : ...) if (%true, %true)
/// ```
/// is canonicalized to
/// ```mlir
/// bufferization.dealloc (%arg0 : ...) if (%true)
/// ```
struct RemoveAllocDeallocPairWhenNoOtherUsers
    : public OpRewritePattern<DeallocOp> {
  using OpRewritePattern<DeallocOp>::OpRewritePattern;
 
  LogicalResult matchAndRewrite(DeallocOp deallocOp,
                                PatternRewriter &rewriter) const override {
    SmallVector<Value> newMemrefs, newConditions;
    SmallVector<Operation *> toDelete;
    for (auto [memref, cond] :
         llvm::zip(deallocOp.getMemrefs(), deallocOp.getConditions())) {
      if (auto allocOp = memref.getDefiningOp<MemoryEffectOpInterface>()) {
        // Check that it is indeed an allocate effect, that the op has no other
        // side effects (which would not allow us to remove the op), and that
        // there are no other users.
        if (allocOp.getEffectOnValue<MemoryEffects::Allocate>(memref) &&
            hasSingleEffect<MemoryEffects::Allocate>(allocOp, memref) &&
            memref.hasOneUse()) {
          toDelete.push_back(allocOp);
          continue;
        }
      }
 
      newMemrefs.push_back(memref);
      newConditions.push_back(cond);
    }
 
    if (failed(updateDeallocIfChanged(deallocOp, newMemrefs, newConditions,
                                      rewriter)))
      return failure();
 
    for (Operation *op : toDelete)
      rewriter.eraseOp(op);
 
    return success();
  }
};
 
} // anonymous namespace
 
void DeallocOp::getCanonicalizationPatterns(RewritePatternSet &results,
                                            MLIRContext *context) {
  populateDeallocOpCanonicalizationPatterns(results, context);
}
 
void bufferization::populateDeallocOpCanonicalizationPatterns(
    RewritePatternSet &patterns, MLIRContext *context) {
  patterns.add<DeallocRemoveDuplicateDeallocMemrefs,
               DeallocRemoveDuplicateRetainedMemrefs, EraseEmptyDealloc,
               EraseAlwaysFalseDealloc, SkipExtractMetadataOfAlloc,
               RemoveAllocDeallocPairWhenNoOtherUsers>(context);
}
 
//===----------------------------------------------------------------------===//
// TableGen'd op method definitions
//===----------------------------------------------------------------------===//
 
#define GET_OP_CLASSES
#include "mlir/Dialect/Bufferization/IR/BufferizationOps.cpp.inc"