20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/TypeSwitch.h"
23 #define DEBUG_TYPE "sroa"
32 if (!getOperation()->getBlock()->isEntryBlock())
35 return {
MemorySlot{getResult(), getElemType()}};
43 void LLVM::AllocaOp::handleBlockArgument(
const MemorySlot &slot,
46 for (
Operation *user : getOperation()->getUsers())
47 if (
auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
48 builder.
create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
49 declareOp.getVarInfo(),
50 declareOp.getLocationExpr());
53 std::optional<PromotableAllocationOpInterface>
54 LLVM::AllocaOp::handlePromotionComplete(
const MemorySlot &slot,
57 if (defaultValue && defaultValue.
use_empty())
67 auto destructurable = dyn_cast<DestructurableTypeInterface>(getElemType());
71 std::optional<DenseMap<Attribute, Type>> destructuredType =
72 destructurable.getSubelementIndexMap();
73 if (!destructuredType)
84 assert(slot.
ptr == getResult());
87 auto destructurableType = cast<DestructurableTypeInterface>(getElemType());
90 Type elemType = destructurableType.getTypeAtIndex(index);
91 assert(elemType &&
"used index must exist");
92 auto subAlloca = builder.
create<LLVM::AllocaOp>(
95 newAllocators.push_back(subAlloca);
96 slotMap.try_emplace<
MemorySlot>(index, {subAlloca.getResult(), elemType});
102 std::optional<DestructurableAllocationOpInterface>
103 LLVM::AllocaOp::handleDestructuringComplete(
105 assert(slot.
ptr == getResult());
114 bool LLVM::LoadOp::loadsFrom(
const MemorySlot &slot) {
115 return getAddr() == slot.
ptr;
118 bool LLVM::LoadOp::storesTo(
const MemorySlot &slot) {
return false; }
122 llvm_unreachable(
"getStored should not be called on LoadOp");
125 bool LLVM::StoreOp::loadsFrom(
const MemorySlot &slot) {
return false; }
127 bool LLVM::StoreOp::storesTo(
const MemorySlot &slot) {
128 return getAddr() == slot.
ptr;
134 if (isa<LLVM::LLVMStructType, LLVM::LLVMArrayType>(type))
137 if (
auto vectorType = dyn_cast<VectorType>(type)) {
139 if (isa<LLVM::LLVMPointerType>(vectorType.getElementType()))
142 return !vectorType.isScalable();
151 Type srcType,
bool narrowingConversion) {
152 if (targetType == srcType)
159 uint64_t targetSize = layout.
getTypeSize(targetType);
164 if (isa<LLVM::LLVMPointerType>(targetType) &&
165 isa<LLVM::LLVMPointerType>(srcType))
166 return targetSize == srcSize;
168 if (narrowingConversion)
169 return targetSize <= srcSize;
170 return targetSize >= srcSize;
175 auto endiannessStr = dyn_cast_or_null<StringAttr>(dataLayout.
getEndianness());
176 return endiannessStr && endiannessStr ==
"big";
185 "expected value to have a convertible type");
187 if (isa<IntegerType>(type))
191 IntegerType valueSizeInteger = builder.
getIntegerType(typeBitSize);
193 if (isa<LLVM::LLVMPointerType>(type))
194 return builder.
createOrFold<LLVM::PtrToIntOp>(loc, valueSizeInteger, val);
195 return builder.
createOrFold<LLVM::BitcastOp>(loc, valueSizeInteger, val);
201 assert(isa<IntegerType>(val.
getType()) &&
202 "expected value to have an integer type");
204 "expected the target type to be supported for conversions");
205 if (val.
getType() == targetType)
207 if (isa<LLVM::LLVMPointerType>(targetType))
208 return builder.
createOrFold<LLVM::IntToPtrOp>(loc, targetType, val);
209 return builder.
createOrFold<LLVM::BitcastOp>(loc, targetType, val);
220 "expected that the compatibility was checked before");
223 if (srcType == targetType)
230 if (isa<LLVM::LLVMPointerType>(targetType) &&
231 isa<LLVM::LLVMPointerType>(srcType))
232 return builder.
createOrFold<LLVM::AddrSpaceCastOp>(loc, targetType,
250 "expected that the compatibility was checked before");
254 if (srcTypeSize == targetTypeSize)
262 uint64_t shiftAmount = srcTypeSize - targetTypeSize;
263 auto shiftConstant = builder.
create<LLVM::ConstantOp>(
266 builder.
createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
269 replacement = builder.
create<LLVM::TruncOp>(
286 "expected that the compatibility was checked before");
289 if (slotTypeSize == valueTypeSize)
302 uint64_t sizeDifference = slotTypeSize - valueTypeSize;
307 Value bigEndianShift = builder.
create<LLVM::ConstantOp>(
310 builder.
createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
319 maskValue = APInt::getAllOnes(sizeDifference).zext(slotTypeSize);
323 maskValue = APInt::getAllOnes(valueTypeSize).zext(slotTypeSize);
324 maskValue.flipAllBits();
343 "expected the reaching definition's type to match the slot's type");
348 bool LLVM::LoadOp::canUsesBeRemoved(
352 if (blockingUses.size() != 1)
354 Value blockingUse = (*blockingUses.begin())->
get();
358 return blockingUse == slot.
ptr && getAddr() == slot.
ptr &&
371 getResult().
getType(), dataLayout);
372 getResult().replaceAllUsesWith(newResult);
376 bool LLVM::StoreOp::canUsesBeRemoved(
380 if (blockingUses.size() != 1)
382 Value blockingUse = (*blockingUses.begin())->
get();
386 return blockingUse == slot.
ptr && getAddr() == slot.
ptr &&
387 getValue() != slot.
ptr &&
408 LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses(
411 return success(getAddr() != slot.
ptr ||
415 LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses(
418 return success(getAddr() != slot.
ptr ||
425 auto subelementIndexMap =
426 cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap();
427 if (!subelementIndexMap)
429 assert(!subelementIndexMap->empty());
432 return subelementIndexMap->lookup(index);
452 usedIndices.insert(index);
461 auto it = subslots.find(index);
462 assert(it != subslots.end());
464 getAddrMutable().set(it->getSecond().ptr);
476 if (getValue() == slot.
ptr)
490 usedIndices.insert(index);
499 auto it = subslots.find(index);
500 assert(it != subslots.end());
502 getAddrMutable().set(it->getSecond().ptr);
515 newBlockingUses.push_back(&use);
519 bool LLVM::BitcastOp::canUsesBeRemoved(
531 bool LLVM::AddrSpaceCastOp::canUsesBeRemoved(
543 bool LLVM::LifetimeStartOp::canUsesBeRemoved(
555 bool LLVM::LifetimeEndOp::canUsesBeRemoved(
567 bool LLVM::InvariantStartOp::canUsesBeRemoved(
574 DeletionKind LLVM::InvariantStartOp::removeBlockingUses(
579 bool LLVM::InvariantEndOp::canUsesBeRemoved(
591 bool LLVM::LaunderInvariantGroupOp::canUsesBeRemoved(
598 DeletionKind LLVM::LaunderInvariantGroupOp::removeBlockingUses(
603 bool LLVM::StripInvariantGroupOp::canUsesBeRemoved(
610 DeletionKind LLVM::StripInvariantGroupOp::removeBlockingUses(
615 bool LLVM::DbgDeclareOp::canUsesBeRemoved(
627 bool LLVM::DbgValueOp::canUsesBeRemoved(
632 if (blockingUses.size() != 1)
635 return (*blockingUses.begin())->get() == getValue();
647 builder.
create<UndefOp>(getValue().getLoc(), getValue().getType());
648 getValueMutable().assign(undef);
652 bool LLVM::DbgDeclareOp::requiresReplacedValues() {
return true; }
654 void LLVM::DbgDeclareOp::visitReplacedValues(
656 for (
auto [op, value] : definitions) {
658 builder.
create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(),
668 return llvm::all_of(gepOp.getIndices(), [](
auto index) {
669 auto indexAttr = llvm::dyn_cast_if_present<IntegerAttr>(index);
670 return indexAttr && indexAttr.getValue() == 0;
674 bool LLVM::GEPOp::canUsesBeRemoved(
695 for (
auto index : gep.getIndices()) {
696 auto constIndex = dyn_cast<IntegerAttr>(index);
699 int64_t gepIndex = constIndex.getInt();
703 indices.push_back(gepIndex);
706 Type currentType = gep.getElemType();
707 uint64_t offset = indices[0] * dataLayout.
getTypeSize(currentType);
709 for (uint64_t index : llvm::drop_begin(indices)) {
712 .Case([&](LLVM::LLVMArrayType arrayType) {
714 index * dataLayout.
getTypeSize(arrayType.getElementType());
715 currentType = arrayType.getElementType();
718 .Case([&](LLVM::LLVMStructType structType) {
720 assert(index < body.size() &&
"expected valid struct indexing");
721 for (uint32_t i : llvm::seq(index)) {
722 if (!structType.isPacked())
723 offset = llvm::alignTo(
729 if (!structType.isPacked())
730 offset = llvm::alignTo(
732 currentType = body[index];
735 .Default([&](
Type type) {
736 LLVM_DEBUG(llvm::dbgs()
737 <<
"[sroa] Unsupported type for offset computations"
752 struct SubslotAccessInfo {
756 uint64_t subslotOffset;
764 static std::optional<SubslotAccessInfo>
766 const DataLayout &dataLayout, LLVM::GEPOp gep) {
774 auto isOutOfBoundsGEPIndex = [](uint64_t index) {
782 .Case([&](LLVM::LLVMArrayType arrayType)
783 -> std::optional<SubslotAccessInfo> {
785 uint64_t elemSize = dataLayout.
getTypeSize(arrayType.getElementType());
786 uint64_t index = *offset / elemSize;
787 if (isOutOfBoundsGEPIndex(index))
789 return SubslotAccessInfo{
static_cast<uint32_t
>(index),
790 *offset - (index * elemSize)};
792 .Case([&](LLVM::LLVMStructType structType)
793 -> std::optional<SubslotAccessInfo> {
794 uint64_t distanceToStart = 0;
799 if (!structType.isPacked()) {
800 distanceToStart = llvm::alignTo(
803 if (offset < distanceToStart)
807 if (offset < distanceToStart + elemSize) {
808 if (isOutOfBoundsGEPIndex(index))
812 return SubslotAccessInfo{
static_cast<uint32_t
>(index),
813 *offset - distanceToStart};
818 distanceToStart += elemSize;
832 LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses(
837 std::optional<uint64_t> gepOffset =
gepToByteOffset(dataLayout, *
this);
842 if (*gepOffset >= slotSize)
860 std::optional<SubslotAccessInfo> accessInfo =
867 usedIndices.insert(indexAttr);
872 uint64_t slotSize = dataLayout.
getTypeSize(subslotType);
873 LLVM::LLVMArrayType remainingSlotType =
875 mustBeSafelyUsed.emplace_back<
MemorySlot>({getRes(), remainingSlotType});
884 std::optional<SubslotAccessInfo> accessInfo =
886 assert(accessInfo &&
"expected access info to be checked before");
889 const MemorySlot &newSlot = subslots.at(indexAttr);
893 getLoc(), getResult().getType(), byteType, newSlot.
ptr,
895 getResult().replaceAllUsesWith(newPtr);
907 template <
class MemIntr>
908 std::optional<uint64_t> getStaticMemIntrLen(MemIntr op) {
912 if (memIntrLen.getBitWidth() > 64)
914 return memIntrLen.getZExtValue();
922 std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemcpyInlineOp op) {
923 APInt memIntrLen = op.getLen();
924 if (memIntrLen.getBitWidth() > 64)
926 return memIntrLen.getZExtValue();
934 std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemsetInlineOp op) {
935 APInt memIntrLen = op.getLen();
936 if (memIntrLen.getBitWidth() > 64)
938 return memIntrLen.getZExtValue();
942 template <
class MemsetIntr>
943 IntegerAttr createMemsetLenAttr(MemsetIntr op) {
944 IntegerAttr memsetLenAttr;
945 bool successfulMatch =
946 matchPattern(op.getLen(), m_Constant<IntegerAttr>(&memsetLenAttr));
947 (void)successfulMatch;
948 assert(successfulMatch);
949 return memsetLenAttr;
956 IntegerAttr createMemsetLenAttr(LLVM::MemsetInlineOp op) {
957 return op.getLenAttr();
963 template <
class MemsetIntr>
964 void createMemsetIntr(
OpBuilder &builder, MemsetIntr toReplace,
965 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
970 void createMemsetIntr(
OpBuilder &builder, LLVM::MemsetOp toReplace,
971 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
974 Value newMemsetSizeValue =
976 .
create<LLVM::ConstantOp>(
977 toReplace.getLen().getLoc(),
981 builder.
create<LLVM::MemsetOp>(toReplace.getLoc(), subslots.at(index).ptr,
982 toReplace.getVal(), newMemsetSizeValue,
983 toReplace.getIsVolatile());
987 void createMemsetIntr(
OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
988 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
991 auto newMemsetSizeValue =
994 builder.
create<LLVM::MemsetInlineOp>(
995 toReplace.getLoc(), subslots.at(index).ptr, toReplace.getVal(),
996 newMemsetSizeValue, toReplace.getIsVolatile());
1003 template <
class MemIntr>
1006 if (!isa<LLVM::LLVMPointerType>(slot.
ptr.
getType()) ||
1007 op.getDst() != slot.
ptr)
1010 std::optional<uint64_t> memIntrLen = getStaticMemIntrLen(op);
1020 auto intIndex = dyn_cast<IntegerAttr>(index);
1021 return intIndex && intIndex.getType() == i32;
1029 template <
class MemsetIntr>
1037 if (op.getIsVolatile())
1040 if (!cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap())
1049 template <
class MemsetIntr>
1054 auto buildMemsetValue = [&](
unsigned width) ->
Value {
1055 assert(width % 8 == 0);
1060 IntegerAttr constantPattern;
1062 assert(constantPattern.getValue().getBitWidth() == 8);
1063 APInt memsetVal(width, 0);
1064 for (
unsigned loBit = 0; loBit < width; loBit += 8)
1065 memsetVal.insertBits(constantPattern.getValue(), loBit);
1066 return builder.
create<LLVM::ConstantOp>(
1076 uint64_t coveredBits = 8;
1077 Value currentValue =
1078 builder.
create<LLVM::ZExtOp>(op.getLoc(), intType, op.getVal());
1079 while (coveredBits < width) {
1081 builder.
create<LLVM::ConstantOp>(op.getLoc(), intType, coveredBits);
1083 builder.
create<LLVM::ShlOp>(op.getLoc(), currentValue, shiftBy);
1085 builder.
create<LLVM::OrOp>(op.getLoc(), currentValue, shifted);
1089 return currentValue;
1092 .Case([&](IntegerType type) ->
Value {
1093 return buildMemsetValue(type.getWidth());
1095 .Case([&](FloatType type) ->
Value {
1096 Value intVal = buildMemsetValue(type.getWidth());
1097 return builder.
create<LLVM::BitcastOp>(op.getLoc(), type, intVal);
1101 "getStored should not be called on memset to unsupported type");
1105 template <
class MemsetIntr>
1111 bool canConvertType =
1113 .Case<IntegerType, FloatType>([](
auto type) {
1114 return type.getWidth() % 8 == 0 && type.getWidth() > 0;
1116 .Default([](
Type) {
return false; });
1117 if (!canConvertType)
1120 if (op.getIsVolatile())
1126 template <
class MemsetIntr>
1132 std::optional<DenseMap<Attribute, Type>> types =
1133 cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap();
1135 IntegerAttr memsetLenAttr = createMemsetLenAttr(op);
1137 bool packed =
false;
1138 if (
auto structType = dyn_cast<LLVM::LLVMStructType>(slot.
elemType))
1139 packed = structType.isPacked();
1142 uint64_t memsetLen = memsetLenAttr.getValue().getZExtValue();
1143 uint64_t covered = 0;
1144 for (
size_t i = 0; i < types->size(); i++) {
1147 Type elemType = types->at(index);
1148 uint64_t typeSize = dataLayout.
getTypeSize(elemType);
1154 if (covered >= memsetLen)
1159 if (subslots.contains(index)) {
1160 uint64_t newMemsetSize =
std::min(memsetLen - covered, typeSize);
1161 createMemsetIntr(builder, op, memsetLenAttr, newMemsetSize, subslots,
1165 covered += typeSize;
1171 bool LLVM::MemsetOp::loadsFrom(
const MemorySlot &slot) {
return false; }
1173 bool LLVM::MemsetOp::storesTo(
const MemorySlot &slot) {
1174 return getDst() == slot.
ptr;
1183 bool LLVM::MemsetOp::canUsesBeRemoved(
1198 LogicalResult LLVM::MemsetOp::ensureOnlySafeAccesses(
1216 return memsetRewire(*
this, slot, subslots, builder, dataLayout);
1219 bool LLVM::MemsetInlineOp::loadsFrom(
const MemorySlot &slot) {
return false; }
1221 bool LLVM::MemsetInlineOp::storesTo(
const MemorySlot &slot) {
1222 return getDst() == slot.
ptr;
1231 bool LLVM::MemsetInlineOp::canUsesBeRemoved(
1246 LogicalResult LLVM::MemsetInlineOp::ensureOnlySafeAccesses(
1252 bool LLVM::MemsetInlineOp::canRewire(
1265 return memsetRewire(*
this, slot, subslots, builder, dataLayout);
1272 template <
class MemcpyLike>
1274 return op.getSrc() == slot.
ptr;
1277 template <
class MemcpyLike>
1279 return op.getDst() == slot.
ptr;
1282 template <
class MemcpyLike>
1285 return builder.
create<LLVM::LoadOp>(op.getLoc(), slot.
elemType, op.getSrc());
1288 template <
class MemcpyLike>
1297 if (op.getDst() == op.getSrc())
1300 if (op.getIsVolatile())
1306 template <
class MemcpyLike>
1311 if (op.loadsFrom(slot))
1312 builder.
create<LLVM::StoreOp>(op.getLoc(), reachingDefinition, op.getDst());
1316 template <
class MemcpyLike>
1317 static LogicalResult
1327 template <
class MemcpyLike>
1332 if (op.getIsVolatile())
1335 if (!cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap())
1345 if (op.getSrc() == slot.
ptr)
1346 usedIndices.insert_range(llvm::make_first_range(slot.
subelementTypes));
1353 template <
class MemcpyLike>
1356 Type toCpy,
bool isVolatile) {
1357 Value memcpySize = builder.
create<LLVM::ConstantOp>(
1360 builder.
create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize,
1366 LLVM::MemcpyInlineOp toReplace,
Value dst,
1367 Value src,
Type toCpy,
bool isVolatile) {
1369 toReplace.getLen().getBitWidth());
1370 builder.
create<LLVM::MemcpyInlineOp>(
1371 toReplace.getLoc(), dst, src,
1379 template <
class MemcpyLike>
1384 if (subslots.empty())
1387 assert((slot.
ptr == op.getDst()) != (slot.
ptr == op.getSrc()));
1388 bool isDst = slot.
ptr == op.getDst();
1391 size_t slotsTreated = 0;
1396 Type indexType = cast<IntegerAttr>(subslots.begin()->first).getType();
1399 if (!subslots.contains(index))
1401 const MemorySlot &subslot = subslots.at(index);
1410 0,
static_cast<int32_t
>(
1411 cast<IntegerAttr>(index).getValue().getZExtValue())};
1412 Value subslotPtrInOther = builder.
create<LLVM::GEPOp>(
1414 isDst ? op.getSrc() : op.getDst(), gepIndices);
1417 createMemcpyLikeToReplace(builder, dataLayout, op,
1418 isDst ? subslot.
ptr : subslotPtrInOther,
1419 isDst ? subslotPtrInOther : subslot.
ptr,
1420 subslot.
elemType, op.getIsVolatile());
1423 assert(subslots.size() == slotsTreated);
1428 bool LLVM::MemcpyOp::loadsFrom(
const MemorySlot &slot) {
1432 bool LLVM::MemcpyOp::storesTo(
const MemorySlot &slot) {
1442 bool LLVM::MemcpyOp::canUsesBeRemoved(
1455 reachingDefinition);
1458 LogicalResult LLVM::MemcpyOp::ensureOnlySafeAccesses(
1476 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1479 bool LLVM::MemcpyInlineOp::loadsFrom(
const MemorySlot &slot) {
1483 bool LLVM::MemcpyInlineOp::storesTo(
const MemorySlot &slot) {
1493 bool LLVM::MemcpyInlineOp::canUsesBeRemoved(
1506 reachingDefinition);
1509 LogicalResult LLVM::MemcpyInlineOp::ensureOnlySafeAccesses(
1515 bool LLVM::MemcpyInlineOp::canRewire(
1528 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1531 bool LLVM::MemmoveOp::loadsFrom(
const MemorySlot &slot) {
1535 bool LLVM::MemmoveOp::storesTo(
const MemorySlot &slot) {
1545 bool LLVM::MemmoveOp::canUsesBeRemoved(
1558 reachingDefinition);
1561 LogicalResult LLVM::MemmoveOp::ensureOnlySafeAccesses(
1579 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1586 std::optional<DenseMap<Attribute, Type>>
1587 LLVM::LLVMStructType::getSubelementIndexMap()
const {
1592 return destructured;
1596 auto indexAttr = llvm::dyn_cast<IntegerAttr>(index);
1597 if (!indexAttr || !indexAttr.getType().isInteger(32))
1599 int32_t indexInt = indexAttr.getInt();
1601 if (indexInt < 0 || body.size() <=
static_cast<uint32_t
>(indexInt))
1603 return body[indexInt];
1606 std::optional<DenseMap<Attribute, Type>>
1607 LLVM::LLVMArrayType::getSubelementIndexMap()
const {
1608 constexpr
size_t maxArraySizeForDestructuring = 16;
1615 for (int32_t index = 0; index < numElements; ++index)
1616 destructured.insert({IntegerAttr::get(i32, index), getElementType()});
1617 return destructured;
1621 auto indexAttr = llvm::dyn_cast<IntegerAttr>(index);
1622 if (!indexAttr || !indexAttr.getType().isInteger(32))
1624 int32_t indexInt = indexAttr.getInt();
1625 if (indexInt < 0 ||
getNumElements() <=
static_cast<uint32_t
>(indexInt))
static Value getBase(Value v)
Looks through known "view-like" ops to find the base memref.
static MLIRContext * getContext(OpFoldResult val)
static int64_t getNumElements(Type t)
Compute the total number of elements in the given type, also taking into account nested types.
static LLVM::LLVMArrayType getByteArrayType(MLIRContext *context, unsigned size)
Constructs a byte array type of the given size.
static LogicalResult memcpyEnsureOnlySafeAccesses(MemcpyLike op, const MemorySlot &slot, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed)
static std::optional< uint64_t > gepToByteOffset(const DataLayout &dataLayout, LLVM::GEPOp gep)
Returns the amount of bytes the provided GEP elements will offset the pointer by.
static bool areAllIndicesI32(const DestructurableMemorySlot &slot)
Checks whether all indices are i32.
static Value castToSameSizedInt(OpBuilder &builder, Location loc, Value val, const DataLayout &dataLayout)
Converts a value to an integer type of the same size.
static Value castSameSizedTypes(OpBuilder &builder, Location loc, Value srcValue, Type targetType, const DataLayout &dataLayout)
Constructs operations that convert srcValue into a new value of type targetType.
static std::optional< SubslotAccessInfo > getSubslotAccessInfo(const DestructurableMemorySlot &slot, const DataLayout &dataLayout, LLVM::GEPOp gep)
Computes subslot access information for an access into slot with the given offset.
static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot)
static DeletionKind memsetRewire(MemsetIntr op, const DestructurableMemorySlot &slot, DenseMap< Attribute, MemorySlot > &subslots, OpBuilder &builder, const DataLayout &dataLayout)
static Type getTypeAtIndex(const DestructurableMemorySlot &slot, Attribute index)
Returns the subslot's type at the requested index.
static bool areConversionCompatible(const DataLayout &layout, Type targetType, Type srcType, bool narrowingConversion)
Checks that rhs can be converted to lhs by a sequence of casts and truncations.
static bool forwardToUsers(Operation *op, SmallVectorImpl< OpOperand * > &newBlockingUses)
Conditions the deletion of the operation to the removal of all its uses.
static bool memsetCanUsesBeRemoved(MemsetIntr op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, SmallVectorImpl< OpOperand * > &newBlockingUses, const DataLayout &dataLayout)
static bool memcpyLoadsFrom(MemcpyLike op, const MemorySlot &slot)
static bool isSupportedTypeForConversion(Type type)
Checks if type can be used in any kind of conversion sequences.
static Value createExtractAndCast(OpBuilder &builder, Location loc, Value srcValue, Type targetType, const DataLayout &dataLayout)
Constructs operations that convert srcValue into a new value of type targetType.
static Value createInsertAndCast(OpBuilder &builder, Location loc, Value srcValue, Value reachingDef, const DataLayout &dataLayout)
Constructs operations that insert the bits of srcValue into the "beginning" of reachingDef (beginning...
static DeletionKind memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, OpBuilder &builder, Value reachingDefinition)
static bool memcpyCanUsesBeRemoved(MemcpyLike op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, SmallVectorImpl< OpOperand * > &newBlockingUses, const DataLayout &dataLayout)
static bool isBigEndian(const DataLayout &dataLayout)
Checks if dataLayout describes a little endian layout.
static bool hasAllZeroIndices(LLVM::GEPOp gepOp)
static bool isValidAccessType(const MemorySlot &slot, Type accessType, const DataLayout &dataLayout)
Checks if slot can be accessed through the provided access type.
static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot, OpBuilder &builder)
static Value castIntValueToSameSizedType(OpBuilder &builder, Location loc, Value val, Type targetType)
Converts a value with an integer type to targetType.
static bool memsetCanRewire(MemsetIntr op, const DestructurableMemorySlot &slot, SmallPtrSetImpl< Attribute > &usedIndices, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed, const DataLayout &dataLayout)
static DeletionKind memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot, DenseMap< Attribute, MemorySlot > &subslots, OpBuilder &builder, const DataLayout &dataLayout)
Rewires a memcpy-like operation.
static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot, OpBuilder &builder)
static bool definitelyWritesOnlyWithinSlot(MemIntr op, const MemorySlot &slot, const DataLayout &dataLayout)
Returns whether one can be sure the memory intrinsic does not write outside of the bounds of the give...
static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot, SmallPtrSetImpl< Attribute > &usedIndices, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed, const DataLayout &dataLayout)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static Type getElementType(Type type, ArrayRef< int32_t > indices, function_ref< InFlightDiagnostic(StringRef)> emitErrorFn)
Walks the given type hierarchy with the given indices, potentially down to component granularity,...
Attributes are known-constant values of operations.
This class represents an argument of a Block.
IntegerAttr getIntegerAttr(Type type, int64_t value)
IntegerType getIntegerType(unsigned width)
MLIRContext * getContext() const
The main mechanism for performing data layout queries.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Attribute getEndianness() const
Returns the specified endianness.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
MLIRContext is the top-level object for a collection of MLIR operations.
This class helps build Operations.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
result_range getResults()
void erase()
Remove this operation from its parent block and delete it.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Dialect & getDialect() const
Get the dialect this type is registered to.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
bool use_empty() const
Returns true if this value has no uses.
MLIRContext * getContext() const
Utility to get the associated MLIRContext that this value is defined in.
Type getType() const
Return the type of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
constexpr int kGEPConstantBitWidth
Bit-width of a 'GEPConstantIndex' within GEPArg.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
detail::constant_int_value_binder m_ConstantInt(IntegerAttr::ValueType *bind_value)
Matches a constant holding a scalar/vector/tensor integer (splat) and writes the integer value to bin...
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
DeletionKind
Returned by operation promotion logic requesting the deletion of an operation.
@ Keep
Keep the operation after promotion.
@ Delete
Delete the operation after promotion.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Memory slot attached with information about its destructuring procedure.
DenseMap< Attribute, Type > subelementTypes
Maps an index within the memory slot to the corresponding subelement type.
Represents a slot in memory.
Value ptr
Pointer to the memory slot, used by operations to refer to it.
Type elemType
Type of the value contained in the slot.