20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/TypeSwitch.h"
23 #define DEBUG_TYPE "sroa"
32 if (!getOperation()->getBlock()->isEntryBlock())
35 return {
MemorySlot{getResult(), getElemType()}};
43 void LLVM::AllocaOp::handleBlockArgument(
const MemorySlot &slot,
46 for (
Operation *user : getOperation()->getUsers())
47 if (
auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
48 builder.
create<LLVM::DbgValueOp>(declareOp.getLoc(), argument,
49 declareOp.getVarInfo(),
50 declareOp.getLocationExpr());
53 std::optional<PromotableAllocationOpInterface>
54 LLVM::AllocaOp::handlePromotionComplete(
const MemorySlot &slot,
57 if (defaultValue && defaultValue.
use_empty())
67 auto destructurable = dyn_cast<DestructurableTypeInterface>(getElemType());
71 std::optional<DenseMap<Attribute, Type>> destructuredType =
72 destructurable.getSubelementIndexMap();
73 if (!destructuredType)
84 assert(slot.
ptr == getResult());
87 auto destructurableType = cast<DestructurableTypeInterface>(getElemType());
90 Type elemType = destructurableType.getTypeAtIndex(index);
91 assert(elemType &&
"used index must exist");
92 auto subAlloca = builder.
create<LLVM::AllocaOp>(
95 newAllocators.push_back(subAlloca);
96 slotMap.try_emplace<
MemorySlot>(index, {subAlloca.getResult(), elemType});
102 std::optional<DestructurableAllocationOpInterface>
103 LLVM::AllocaOp::handleDestructuringComplete(
105 assert(slot.
ptr == getResult());
114 bool LLVM::LoadOp::loadsFrom(
const MemorySlot &slot) {
115 return getAddr() == slot.
ptr;
118 bool LLVM::LoadOp::storesTo(
const MemorySlot &slot) {
return false; }
122 llvm_unreachable(
"getStored should not be called on LoadOp");
125 bool LLVM::StoreOp::loadsFrom(
const MemorySlot &slot) {
return false; }
127 bool LLVM::StoreOp::storesTo(
const MemorySlot &slot) {
128 return getAddr() == slot.
ptr;
134 if (isa<LLVM::LLVMStructType, LLVM::LLVMArrayType>(type))
140 if (isa<LLVM::LLVMFixedVectorType, LLVM::LLVMScalableVectorType>(type))
144 if (
auto vectorType = dyn_cast<VectorType>(type))
145 return !vectorType.isScalable();
153 Type srcType,
bool narrowingConversion) {
154 if (targetType == srcType)
161 uint64_t targetSize = layout.
getTypeSize(targetType);
166 if (isa<LLVM::LLVMPointerType>(targetType) &&
167 isa<LLVM::LLVMPointerType>(srcType))
168 return targetSize == srcSize;
170 if (narrowingConversion)
171 return targetSize <= srcSize;
172 return targetSize >= srcSize;
177 auto endiannessStr = dyn_cast_or_null<StringAttr>(dataLayout.
getEndianness());
178 return endiannessStr && endiannessStr ==
"big";
187 "expected value to have a convertible type");
189 if (isa<IntegerType>(type))
193 IntegerType valueSizeInteger = builder.
getIntegerType(typeBitSize);
195 if (isa<LLVM::LLVMPointerType>(type))
196 return builder.
createOrFold<LLVM::PtrToIntOp>(loc, valueSizeInteger, val);
197 return builder.
createOrFold<LLVM::BitcastOp>(loc, valueSizeInteger, val);
203 assert(isa<IntegerType>(val.
getType()) &&
204 "expected value to have an integer type");
206 "expected the target type to be supported for conversions");
207 if (val.
getType() == targetType)
209 if (isa<LLVM::LLVMPointerType>(targetType))
210 return builder.
createOrFold<LLVM::IntToPtrOp>(loc, targetType, val);
211 return builder.
createOrFold<LLVM::BitcastOp>(loc, targetType, val);
222 "expected that the compatibility was checked before");
225 if (srcType == targetType)
232 if (isa<LLVM::LLVMPointerType>(targetType) &&
233 isa<LLVM::LLVMPointerType>(srcType))
234 return builder.
createOrFold<LLVM::AddrSpaceCastOp>(loc, targetType,
252 "expected that the compatibility was checked before");
256 if (srcTypeSize == targetTypeSize)
264 uint64_t shiftAmount = srcTypeSize - targetTypeSize;
265 auto shiftConstant = builder.
create<LLVM::ConstantOp>(
268 builder.
createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
271 replacement = builder.
create<LLVM::TruncOp>(
288 "expected that the compatibility was checked before");
291 if (slotTypeSize == valueTypeSize)
304 uint64_t sizeDifference = slotTypeSize - valueTypeSize;
309 Value bigEndianShift = builder.
create<LLVM::ConstantOp>(
312 builder.
createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
321 maskValue = APInt::getAllOnes(sizeDifference).zext(slotTypeSize);
325 maskValue = APInt::getAllOnes(valueTypeSize).zext(slotTypeSize);
326 maskValue.flipAllBits();
345 "expected the reaching definition's type to match the slot's type");
350 bool LLVM::LoadOp::canUsesBeRemoved(
354 if (blockingUses.size() != 1)
356 Value blockingUse = (*blockingUses.begin())->
get();
360 return blockingUse == slot.
ptr && getAddr() == slot.
ptr &&
373 getResult().
getType(), dataLayout);
374 getResult().replaceAllUsesWith(newResult);
378 bool LLVM::StoreOp::canUsesBeRemoved(
382 if (blockingUses.size() != 1)
384 Value blockingUse = (*blockingUses.begin())->
get();
388 return blockingUse == slot.
ptr && getAddr() == slot.
ptr &&
389 getValue() != slot.
ptr &&
410 LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses(
413 return success(getAddr() != slot.
ptr ||
417 LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses(
420 return success(getAddr() != slot.
ptr ||
427 auto subelementIndexMap =
428 cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap();
429 if (!subelementIndexMap)
431 assert(!subelementIndexMap->empty());
434 return subelementIndexMap->lookup(index);
454 usedIndices.insert(index);
463 auto it = subslots.find(index);
464 assert(it != subslots.end());
466 getAddrMutable().set(it->getSecond().ptr);
478 if (getValue() == slot.
ptr)
492 usedIndices.insert(index);
501 auto it = subslots.find(index);
502 assert(it != subslots.end());
504 getAddrMutable().set(it->getSecond().ptr);
517 newBlockingUses.push_back(&use);
521 bool LLVM::BitcastOp::canUsesBeRemoved(
533 bool LLVM::AddrSpaceCastOp::canUsesBeRemoved(
545 bool LLVM::LifetimeStartOp::canUsesBeRemoved(
557 bool LLVM::LifetimeEndOp::canUsesBeRemoved(
569 bool LLVM::InvariantStartOp::canUsesBeRemoved(
576 DeletionKind LLVM::InvariantStartOp::removeBlockingUses(
581 bool LLVM::InvariantEndOp::canUsesBeRemoved(
593 bool LLVM::LaunderInvariantGroupOp::canUsesBeRemoved(
600 DeletionKind LLVM::LaunderInvariantGroupOp::removeBlockingUses(
605 bool LLVM::StripInvariantGroupOp::canUsesBeRemoved(
612 DeletionKind LLVM::StripInvariantGroupOp::removeBlockingUses(
617 bool LLVM::DbgDeclareOp::canUsesBeRemoved(
629 bool LLVM::DbgValueOp::canUsesBeRemoved(
634 if (blockingUses.size() != 1)
637 return (*blockingUses.begin())->get() == getValue();
649 builder.
create<UndefOp>(getValue().getLoc(), getValue().getType());
650 getValueMutable().assign(undef);
654 bool LLVM::DbgDeclareOp::requiresReplacedValues() {
return true; }
656 void LLVM::DbgDeclareOp::visitReplacedValues(
658 for (
auto [op, value] : definitions) {
660 builder.
create<LLVM::DbgValueOp>(getLoc(), value, getVarInfo(),
670 return llvm::all_of(gepOp.getIndices(), [](
auto index) {
671 auto indexAttr = llvm::dyn_cast_if_present<IntegerAttr>(index);
672 return indexAttr && indexAttr.getValue() == 0;
676 bool LLVM::GEPOp::canUsesBeRemoved(
697 for (
auto index : gep.getIndices()) {
698 auto constIndex = dyn_cast<IntegerAttr>(index);
701 int64_t gepIndex = constIndex.getInt();
705 indices.push_back(gepIndex);
708 Type currentType = gep.getElemType();
709 uint64_t offset = indices[0] * dataLayout.
getTypeSize(currentType);
711 for (uint64_t index : llvm::drop_begin(indices)) {
714 .Case([&](LLVM::LLVMArrayType arrayType) {
716 index * dataLayout.
getTypeSize(arrayType.getElementType());
717 currentType = arrayType.getElementType();
720 .Case([&](LLVM::LLVMStructType structType) {
722 assert(index < body.size() &&
"expected valid struct indexing");
723 for (uint32_t i : llvm::seq(index)) {
724 if (!structType.isPacked())
725 offset = llvm::alignTo(
731 if (!structType.isPacked())
732 offset = llvm::alignTo(
734 currentType = body[index];
737 .Default([&](
Type type) {
738 LLVM_DEBUG(llvm::dbgs()
739 <<
"[sroa] Unsupported type for offset computations"
754 struct SubslotAccessInfo {
758 uint64_t subslotOffset;
766 static std::optional<SubslotAccessInfo>
768 const DataLayout &dataLayout, LLVM::GEPOp gep) {
776 auto isOutOfBoundsGEPIndex = [](uint64_t index) {
784 .Case([&](LLVM::LLVMArrayType arrayType)
785 -> std::optional<SubslotAccessInfo> {
787 uint64_t elemSize = dataLayout.
getTypeSize(arrayType.getElementType());
788 uint64_t index = *offset / elemSize;
789 if (isOutOfBoundsGEPIndex(index))
791 return SubslotAccessInfo{
static_cast<uint32_t
>(index),
792 *offset - (index * elemSize)};
794 .Case([&](LLVM::LLVMStructType structType)
795 -> std::optional<SubslotAccessInfo> {
796 uint64_t distanceToStart = 0;
801 if (!structType.isPacked()) {
802 distanceToStart = llvm::alignTo(
805 if (offset < distanceToStart)
809 if (offset < distanceToStart + elemSize) {
810 if (isOutOfBoundsGEPIndex(index))
814 return SubslotAccessInfo{
static_cast<uint32_t
>(index),
815 *offset - distanceToStart};
820 distanceToStart += elemSize;
834 LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses(
839 std::optional<uint64_t> gepOffset =
gepToByteOffset(dataLayout, *
this);
844 if (*gepOffset >= slotSize)
862 std::optional<SubslotAccessInfo> accessInfo =
869 usedIndices.insert(indexAttr);
874 uint64_t slotSize = dataLayout.
getTypeSize(subslotType);
875 LLVM::LLVMArrayType remainingSlotType =
877 mustBeSafelyUsed.emplace_back<
MemorySlot>({getRes(), remainingSlotType});
886 std::optional<SubslotAccessInfo> accessInfo =
888 assert(accessInfo &&
"expected access info to be checked before");
891 const MemorySlot &newSlot = subslots.at(indexAttr);
895 getLoc(), getResult().getType(), byteType, newSlot.
ptr,
897 getResult().replaceAllUsesWith(newPtr);
909 template <
class MemIntr>
910 std::optional<uint64_t> getStaticMemIntrLen(MemIntr op) {
914 if (memIntrLen.getBitWidth() > 64)
916 return memIntrLen.getZExtValue();
924 std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemcpyInlineOp op) {
925 APInt memIntrLen = op.getLen();
926 if (memIntrLen.getBitWidth() > 64)
928 return memIntrLen.getZExtValue();
936 std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemsetInlineOp op) {
937 APInt memIntrLen = op.getLen();
938 if (memIntrLen.getBitWidth() > 64)
940 return memIntrLen.getZExtValue();
944 template <
class MemsetIntr>
945 IntegerAttr createMemsetLenAttr(MemsetIntr op) {
946 IntegerAttr memsetLenAttr;
947 bool successfulMatch =
948 matchPattern(op.getLen(), m_Constant<IntegerAttr>(&memsetLenAttr));
949 (void)successfulMatch;
950 assert(successfulMatch);
951 return memsetLenAttr;
958 IntegerAttr createMemsetLenAttr(LLVM::MemsetInlineOp op) {
959 return op.getLenAttr();
965 template <
class MemsetIntr>
966 void createMemsetIntr(
OpBuilder &builder, MemsetIntr toReplace,
967 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
972 void createMemsetIntr(
OpBuilder &builder, LLVM::MemsetOp toReplace,
973 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
976 Value newMemsetSizeValue =
978 .
create<LLVM::ConstantOp>(
979 toReplace.getLen().getLoc(),
983 builder.
create<LLVM::MemsetOp>(toReplace.getLoc(), subslots.at(index).ptr,
984 toReplace.getVal(), newMemsetSizeValue,
985 toReplace.getIsVolatile());
989 void createMemsetIntr(
OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
990 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
993 auto newMemsetSizeValue =
996 builder.
create<LLVM::MemsetInlineOp>(
997 toReplace.getLoc(), subslots.at(index).ptr, toReplace.getVal(),
998 newMemsetSizeValue, toReplace.getIsVolatile());
1005 template <
class MemIntr>
1008 if (!isa<LLVM::LLVMPointerType>(slot.
ptr.
getType()) ||
1009 op.getDst() != slot.
ptr)
1012 std::optional<uint64_t> memIntrLen = getStaticMemIntrLen(op);
1022 auto intIndex = dyn_cast<IntegerAttr>(index);
1023 return intIndex && intIndex.getType() == i32;
1031 template <
class MemsetIntr>
1039 if (op.getIsVolatile())
1042 if (!cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap())
1051 template <
class MemsetIntr>
1056 auto buildMemsetValue = [&](
unsigned width) ->
Value {
1057 assert(width % 8 == 0);
1062 IntegerAttr constantPattern;
1064 assert(constantPattern.getValue().getBitWidth() == 8);
1065 APInt memsetVal(width, 0);
1066 for (
unsigned loBit = 0; loBit < width; loBit += 8)
1067 memsetVal.insertBits(constantPattern.getValue(), loBit);
1068 return builder.
create<LLVM::ConstantOp>(
1078 uint64_t coveredBits = 8;
1079 Value currentValue =
1080 builder.
create<LLVM::ZExtOp>(op.getLoc(), intType, op.getVal());
1081 while (coveredBits < width) {
1083 builder.
create<LLVM::ConstantOp>(op.getLoc(), intType, coveredBits);
1085 builder.
create<LLVM::ShlOp>(op.getLoc(), currentValue, shiftBy);
1087 builder.
create<LLVM::OrOp>(op.getLoc(), currentValue, shifted);
1091 return currentValue;
1094 .Case([&](IntegerType type) ->
Value {
1095 return buildMemsetValue(type.getWidth());
1097 .Case([&](FloatType type) ->
Value {
1098 Value intVal = buildMemsetValue(type.getWidth());
1099 return builder.
create<LLVM::BitcastOp>(op.getLoc(), type, intVal);
1103 "getStored should not be called on memset to unsupported type");
1107 template <
class MemsetIntr>
1113 bool canConvertType =
1115 .Case<IntegerType, FloatType>([](
auto type) {
1116 return type.getWidth() % 8 == 0 && type.getWidth() > 0;
1118 .Default([](
Type) {
return false; });
1119 if (!canConvertType)
1122 if (op.getIsVolatile())
1128 template <
class MemsetIntr>
1134 std::optional<DenseMap<Attribute, Type>> types =
1135 cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap();
1137 IntegerAttr memsetLenAttr = createMemsetLenAttr(op);
1139 bool packed =
false;
1140 if (
auto structType = dyn_cast<LLVM::LLVMStructType>(slot.
elemType))
1141 packed = structType.isPacked();
1144 uint64_t memsetLen = memsetLenAttr.getValue().getZExtValue();
1145 uint64_t covered = 0;
1146 for (
size_t i = 0; i < types->size(); i++) {
1149 Type elemType = types->at(index);
1150 uint64_t typeSize = dataLayout.
getTypeSize(elemType);
1156 if (covered >= memsetLen)
1161 if (subslots.contains(index)) {
1162 uint64_t newMemsetSize =
std::min(memsetLen - covered, typeSize);
1163 createMemsetIntr(builder, op, memsetLenAttr, newMemsetSize, subslots,
1167 covered += typeSize;
1173 bool LLVM::MemsetOp::loadsFrom(
const MemorySlot &slot) {
return false; }
1175 bool LLVM::MemsetOp::storesTo(
const MemorySlot &slot) {
1176 return getDst() == slot.
ptr;
1185 bool LLVM::MemsetOp::canUsesBeRemoved(
1200 LogicalResult LLVM::MemsetOp::ensureOnlySafeAccesses(
1218 return memsetRewire(*
this, slot, subslots, builder, dataLayout);
1221 bool LLVM::MemsetInlineOp::loadsFrom(
const MemorySlot &slot) {
return false; }
1223 bool LLVM::MemsetInlineOp::storesTo(
const MemorySlot &slot) {
1224 return getDst() == slot.
ptr;
1233 bool LLVM::MemsetInlineOp::canUsesBeRemoved(
1248 LogicalResult LLVM::MemsetInlineOp::ensureOnlySafeAccesses(
1254 bool LLVM::MemsetInlineOp::canRewire(
1267 return memsetRewire(*
this, slot, subslots, builder, dataLayout);
1274 template <
class MemcpyLike>
1276 return op.getSrc() == slot.
ptr;
1279 template <
class MemcpyLike>
1281 return op.getDst() == slot.
ptr;
1284 template <
class MemcpyLike>
1287 return builder.
create<LLVM::LoadOp>(op.getLoc(), slot.
elemType, op.getSrc());
1290 template <
class MemcpyLike>
1299 if (op.getDst() == op.getSrc())
1302 if (op.getIsVolatile())
1308 template <
class MemcpyLike>
1313 if (op.loadsFrom(slot))
1314 builder.
create<LLVM::StoreOp>(op.getLoc(), reachingDefinition, op.getDst());
1318 template <
class MemcpyLike>
1319 static LogicalResult
1329 template <
class MemcpyLike>
1334 if (op.getIsVolatile())
1337 if (!cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap())
1347 if (op.getSrc() == slot.
ptr)
1349 usedIndices.insert(index);
1356 template <
class MemcpyLike>
1359 Type toCpy,
bool isVolatile) {
1360 Value memcpySize = builder.
create<LLVM::ConstantOp>(
1363 builder.
create<MemcpyLike>(toReplace.getLoc(), dst, src, memcpySize,
1369 LLVM::MemcpyInlineOp toReplace,
Value dst,
1370 Value src,
Type toCpy,
bool isVolatile) {
1372 toReplace.getLen().getBitWidth());
1373 builder.
create<LLVM::MemcpyInlineOp>(
1374 toReplace.getLoc(), dst, src,
1382 template <
class MemcpyLike>
1387 if (subslots.empty())
1390 assert((slot.
ptr == op.getDst()) != (slot.
ptr == op.getSrc()));
1391 bool isDst = slot.
ptr == op.getDst();
1394 size_t slotsTreated = 0;
1399 Type indexType = cast<IntegerAttr>(subslots.begin()->first).getType();
1402 if (!subslots.contains(index))
1404 const MemorySlot &subslot = subslots.at(index);
1413 0,
static_cast<int32_t
>(
1414 cast<IntegerAttr>(index).getValue().getZExtValue())};
1415 Value subslotPtrInOther = builder.
create<LLVM::GEPOp>(
1417 isDst ? op.getSrc() : op.getDst(), gepIndices);
1420 createMemcpyLikeToReplace(builder, dataLayout, op,
1421 isDst ? subslot.
ptr : subslotPtrInOther,
1422 isDst ? subslotPtrInOther : subslot.
ptr,
1423 subslot.
elemType, op.getIsVolatile());
1426 assert(subslots.size() == slotsTreated);
1431 bool LLVM::MemcpyOp::loadsFrom(
const MemorySlot &slot) {
1435 bool LLVM::MemcpyOp::storesTo(
const MemorySlot &slot) {
1445 bool LLVM::MemcpyOp::canUsesBeRemoved(
1458 reachingDefinition);
1461 LogicalResult LLVM::MemcpyOp::ensureOnlySafeAccesses(
1479 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1482 bool LLVM::MemcpyInlineOp::loadsFrom(
const MemorySlot &slot) {
1486 bool LLVM::MemcpyInlineOp::storesTo(
const MemorySlot &slot) {
1496 bool LLVM::MemcpyInlineOp::canUsesBeRemoved(
1509 reachingDefinition);
1512 LogicalResult LLVM::MemcpyInlineOp::ensureOnlySafeAccesses(
1518 bool LLVM::MemcpyInlineOp::canRewire(
1531 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1534 bool LLVM::MemmoveOp::loadsFrom(
const MemorySlot &slot) {
1538 bool LLVM::MemmoveOp::storesTo(
const MemorySlot &slot) {
1548 bool LLVM::MemmoveOp::canUsesBeRemoved(
1561 reachingDefinition);
1564 LogicalResult LLVM::MemmoveOp::ensureOnlySafeAccesses(
1582 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1589 std::optional<DenseMap<Attribute, Type>>
1590 LLVM::LLVMStructType::getSubelementIndexMap()
const {
1595 return destructured;
1599 auto indexAttr = llvm::dyn_cast<IntegerAttr>(index);
1600 if (!indexAttr || !indexAttr.getType().isInteger(32))
1602 int32_t indexInt = indexAttr.getInt();
1604 if (indexInt < 0 || body.size() <=
static_cast<uint32_t
>(indexInt))
1606 return body[indexInt];
1609 std::optional<DenseMap<Attribute, Type>>
1610 LLVM::LLVMArrayType::getSubelementIndexMap()
const {
1611 constexpr
size_t maxArraySizeForDestructuring = 16;
1618 for (int32_t index = 0; index < numElements; ++index)
1619 destructured.insert({IntegerAttr::get(i32, index), getElementType()});
1620 return destructured;
1624 auto indexAttr = llvm::dyn_cast<IntegerAttr>(index);
1625 if (!indexAttr || !indexAttr.getType().isInteger(32))
1627 int32_t indexInt = indexAttr.getInt();
1628 if (indexInt < 0 ||
getNumElements() <=
static_cast<uint32_t
>(indexInt))
static Value getBase(Value v)
Looks through known "view-like" ops to find the base memref.
static MLIRContext * getContext(OpFoldResult val)
static int64_t getNumElements(Type t)
Compute the total number of elements in the given type, also taking into account nested types.
static LLVM::LLVMArrayType getByteArrayType(MLIRContext *context, unsigned size)
Constructs a byte array type of the given size.
static LogicalResult memcpyEnsureOnlySafeAccesses(MemcpyLike op, const MemorySlot &slot, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed)
static std::optional< uint64_t > gepToByteOffset(const DataLayout &dataLayout, LLVM::GEPOp gep)
Returns the amount of bytes the provided GEP elements will offset the pointer by.
static bool areAllIndicesI32(const DestructurableMemorySlot &slot)
Checks whether all indices are i32.
static Value castToSameSizedInt(OpBuilder &builder, Location loc, Value val, const DataLayout &dataLayout)
Converts a value to an integer type of the same size.
static Value castSameSizedTypes(OpBuilder &builder, Location loc, Value srcValue, Type targetType, const DataLayout &dataLayout)
Constructs operations that convert srcValue into a new value of type targetType.
static std::optional< SubslotAccessInfo > getSubslotAccessInfo(const DestructurableMemorySlot &slot, const DataLayout &dataLayout, LLVM::GEPOp gep)
Computes subslot access information for an access into slot with the given offset.
static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot)
static DeletionKind memsetRewire(MemsetIntr op, const DestructurableMemorySlot &slot, DenseMap< Attribute, MemorySlot > &subslots, OpBuilder &builder, const DataLayout &dataLayout)
static Type getTypeAtIndex(const DestructurableMemorySlot &slot, Attribute index)
Returns the subslot's type at the requested index.
static bool areConversionCompatible(const DataLayout &layout, Type targetType, Type srcType, bool narrowingConversion)
Checks that rhs can be converted to lhs by a sequence of casts and truncations.
static bool forwardToUsers(Operation *op, SmallVectorImpl< OpOperand * > &newBlockingUses)
Conditions the deletion of the operation to the removal of all its uses.
static bool memsetCanUsesBeRemoved(MemsetIntr op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, SmallVectorImpl< OpOperand * > &newBlockingUses, const DataLayout &dataLayout)
static bool memcpyLoadsFrom(MemcpyLike op, const MemorySlot &slot)
static bool isSupportedTypeForConversion(Type type)
Checks if type can be used in any kind of conversion sequences.
static Value createExtractAndCast(OpBuilder &builder, Location loc, Value srcValue, Type targetType, const DataLayout &dataLayout)
Constructs operations that convert srcValue into a new value of type targetType.
static Value createInsertAndCast(OpBuilder &builder, Location loc, Value srcValue, Value reachingDef, const DataLayout &dataLayout)
Constructs operations that insert the bits of srcValue into the "beginning" of reachingDef (beginning...
static DeletionKind memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, OpBuilder &builder, Value reachingDefinition)
static bool memcpyCanUsesBeRemoved(MemcpyLike op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, SmallVectorImpl< OpOperand * > &newBlockingUses, const DataLayout &dataLayout)
static bool isBigEndian(const DataLayout &dataLayout)
Checks if dataLayout describes a little endian layout.
static bool hasAllZeroIndices(LLVM::GEPOp gepOp)
static bool isValidAccessType(const MemorySlot &slot, Type accessType, const DataLayout &dataLayout)
Checks if slot can be accessed through the provided access type.
static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot, OpBuilder &builder)
static Value castIntValueToSameSizedType(OpBuilder &builder, Location loc, Value val, Type targetType)
Converts a value with an integer type to targetType.
static bool memsetCanRewire(MemsetIntr op, const DestructurableMemorySlot &slot, SmallPtrSetImpl< Attribute > &usedIndices, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed, const DataLayout &dataLayout)
static DeletionKind memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot, DenseMap< Attribute, MemorySlot > &subslots, OpBuilder &builder, const DataLayout &dataLayout)
Rewires a memcpy-like operation.
static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot, OpBuilder &builder)
static bool definitelyWritesOnlyWithinSlot(MemIntr op, const MemorySlot &slot, const DataLayout &dataLayout)
Returns whether one can be sure the memory intrinsic does not write outside of the bounds of the give...
static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot, SmallPtrSetImpl< Attribute > &usedIndices, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed, const DataLayout &dataLayout)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static Type getElementType(Type type, ArrayRef< int32_t > indices, function_ref< InFlightDiagnostic(StringRef)> emitErrorFn)
Walks the given type hierarchy with the given indices, potentially down to component granularity,...
Attributes are known-constant values of operations.
This class represents an argument of a Block.
IntegerAttr getIntegerAttr(Type type, int64_t value)
IntegerType getIntegerType(unsigned width)
MLIRContext * getContext() const
The main mechanism for performing data layout queries.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Attribute getEndianness() const
Returns the specified endianness.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
MLIRContext is the top-level object for a collection of MLIR operations.
This class helps build Operations.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
result_range getResults()
void erase()
Remove this operation from its parent block and delete it.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Dialect & getDialect() const
Get the dialect this type is registered to.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
bool use_empty() const
Returns true if this value has no uses.
MLIRContext * getContext() const
Utility to get the associated MLIRContext that this value is defined in.
Type getType() const
Return the type of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
constexpr int kGEPConstantBitWidth
Bit-width of a 'GEPConstantIndex' within GEPArg.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
detail::constant_int_value_binder m_ConstantInt(IntegerAttr::ValueType *bind_value)
Matches a constant holding a scalar/vector/tensor integer (splat) and writes the integer value to bin...
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
DeletionKind
Returned by operation promotion logic requesting the deletion of an operation.
@ Keep
Keep the operation after promotion.
@ Delete
Delete the operation after promotion.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Memory slot attached with information about its destructuring procedure.
DenseMap< Attribute, Type > subelementTypes
Maps an index within the memory slot to the corresponding subelement type.
Represents a slot in memory.
Value ptr
Pointer to the memory slot, used by operations to refer to it.
Type elemType
Type of the value contained in the slot.