20 #include "llvm/ADT/STLExtras.h"
21 #include "llvm/ADT/TypeSwitch.h"
22 #include "llvm/Support/DebugLog.h"
24 #define DEBUG_TYPE "sroa"
33 if (!getOperation()->getBlock()->isEntryBlock())
36 return {
MemorySlot{getResult(), getElemType()}};
41 return LLVM::UndefOp::create(builder, getLoc(), slot.
elemType);
44 void LLVM::AllocaOp::handleBlockArgument(
const MemorySlot &slot,
47 for (
Operation *user : getOperation()->getUsers())
48 if (
auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
49 LLVM::DbgValueOp::create(builder, declareOp.getLoc(), argument,
50 declareOp.getVarInfo(),
51 declareOp.getLocationExpr());
54 std::optional<PromotableAllocationOpInterface>
55 LLVM::AllocaOp::handlePromotionComplete(
const MemorySlot &slot,
58 if (defaultValue && defaultValue.
use_empty())
68 auto destructurable = dyn_cast<DestructurableTypeInterface>(getElemType());
72 std::optional<DenseMap<Attribute, Type>> destructuredType =
73 destructurable.getSubelementIndexMap();
74 if (!destructuredType)
85 assert(slot.
ptr == getResult());
88 auto destructurableType = cast<DestructurableTypeInterface>(getElemType());
91 Type elemType = destructurableType.getTypeAtIndex(index);
92 assert(elemType &&
"used index must exist");
93 auto subAlloca = LLVM::AllocaOp::create(
96 newAllocators.push_back(subAlloca);
97 slotMap.try_emplace<
MemorySlot>(index, {subAlloca.getResult(), elemType});
103 std::optional<DestructurableAllocationOpInterface>
104 LLVM::AllocaOp::handleDestructuringComplete(
106 assert(slot.
ptr == getResult());
115 bool LLVM::LoadOp::loadsFrom(
const MemorySlot &slot) {
116 return getAddr() == slot.
ptr;
119 bool LLVM::LoadOp::storesTo(
const MemorySlot &slot) {
return false; }
123 llvm_unreachable(
"getStored should not be called on LoadOp");
126 bool LLVM::StoreOp::loadsFrom(
const MemorySlot &slot) {
return false; }
128 bool LLVM::StoreOp::storesTo(
const MemorySlot &slot) {
129 return getAddr() == slot.
ptr;
135 if (isa<LLVM::LLVMStructType, LLVM::LLVMArrayType>(type))
138 if (
auto vectorType = dyn_cast<VectorType>(type)) {
140 if (isa<LLVM::LLVMPointerType>(vectorType.getElementType()))
143 return !vectorType.isScalable();
152 Type srcType,
bool narrowingConversion) {
153 if (targetType == srcType)
160 uint64_t targetSize = layout.
getTypeSize(targetType);
165 if (isa<LLVM::LLVMPointerType>(targetType) &&
166 isa<LLVM::LLVMPointerType>(srcType))
167 return targetSize == srcSize;
169 if (narrowingConversion)
170 return targetSize <= srcSize;
171 return targetSize >= srcSize;
176 auto endiannessStr = dyn_cast_or_null<StringAttr>(dataLayout.
getEndianness());
177 return endiannessStr && endiannessStr ==
"big";
186 "expected value to have a convertible type");
188 if (isa<IntegerType>(type))
192 IntegerType valueSizeInteger = builder.
getIntegerType(typeBitSize);
194 if (isa<LLVM::LLVMPointerType>(type))
195 return builder.
createOrFold<LLVM::PtrToIntOp>(loc, valueSizeInteger, val);
196 return builder.
createOrFold<LLVM::BitcastOp>(loc, valueSizeInteger, val);
202 assert(isa<IntegerType>(val.
getType()) &&
203 "expected value to have an integer type");
205 "expected the target type to be supported for conversions");
206 if (val.
getType() == targetType)
208 if (isa<LLVM::LLVMPointerType>(targetType))
209 return builder.
createOrFold<LLVM::IntToPtrOp>(loc, targetType, val);
210 return builder.
createOrFold<LLVM::BitcastOp>(loc, targetType, val);
221 "expected that the compatibility was checked before");
224 if (srcType == targetType)
231 if (isa<LLVM::LLVMPointerType>(targetType) &&
232 isa<LLVM::LLVMPointerType>(srcType))
233 return builder.
createOrFold<LLVM::AddrSpaceCastOp>(loc, targetType,
251 "expected that the compatibility was checked before");
255 if (srcTypeSize == targetTypeSize)
263 uint64_t shiftAmount = srcTypeSize - targetTypeSize;
264 auto shiftConstant = LLVM::ConstantOp::create(
267 builder.
createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
270 replacement = LLVM::TruncOp::create(
271 builder, loc, builder.
getIntegerType(targetTypeSize), replacement);
287 "expected that the compatibility was checked before");
290 if (slotTypeSize == valueTypeSize)
303 uint64_t sizeDifference = slotTypeSize - valueTypeSize;
308 Value bigEndianShift = LLVM::ConstantOp::create(
312 builder.
createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
321 maskValue = APInt::getAllOnes(sizeDifference).zext(slotTypeSize);
325 maskValue = APInt::getAllOnes(valueTypeSize).zext(slotTypeSize);
326 maskValue.flipAllBits();
330 Value mask = LLVM::ConstantOp::create(
345 "expected the reaching definition's type to match the slot's type");
350 bool LLVM::LoadOp::canUsesBeRemoved(
354 if (blockingUses.size() != 1)
356 Value blockingUse = (*blockingUses.begin())->
get();
360 return blockingUse == slot.
ptr && getAddr() == slot.
ptr &&
373 getResult().
getType(), dataLayout);
374 getResult().replaceAllUsesWith(newResult);
378 bool LLVM::StoreOp::canUsesBeRemoved(
382 if (blockingUses.size() != 1)
384 Value blockingUse = (*blockingUses.begin())->
get();
388 return blockingUse == slot.
ptr && getAddr() == slot.
ptr &&
389 getValue() != slot.
ptr &&
410 LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses(
413 return success(getAddr() != slot.
ptr ||
417 LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses(
420 return success(getAddr() != slot.
ptr ||
427 auto subelementIndexMap =
428 cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap();
429 if (!subelementIndexMap)
431 assert(!subelementIndexMap->empty());
434 return subelementIndexMap->lookup(index);
454 usedIndices.insert(index);
463 auto it = subslots.find(index);
464 assert(it != subslots.end());
466 getAddrMutable().set(it->getSecond().ptr);
478 if (getValue() == slot.
ptr)
492 usedIndices.insert(index);
501 auto it = subslots.find(index);
502 assert(it != subslots.end());
504 getAddrMutable().set(it->getSecond().ptr);
517 newBlockingUses.push_back(&use);
521 bool LLVM::BitcastOp::canUsesBeRemoved(
533 bool LLVM::AddrSpaceCastOp::canUsesBeRemoved(
545 bool LLVM::LifetimeStartOp::canUsesBeRemoved(
557 bool LLVM::LifetimeEndOp::canUsesBeRemoved(
569 bool LLVM::InvariantStartOp::canUsesBeRemoved(
576 DeletionKind LLVM::InvariantStartOp::removeBlockingUses(
581 bool LLVM::InvariantEndOp::canUsesBeRemoved(
593 bool LLVM::LaunderInvariantGroupOp::canUsesBeRemoved(
600 DeletionKind LLVM::LaunderInvariantGroupOp::removeBlockingUses(
605 bool LLVM::StripInvariantGroupOp::canUsesBeRemoved(
612 DeletionKind LLVM::StripInvariantGroupOp::removeBlockingUses(
617 bool LLVM::DbgDeclareOp::canUsesBeRemoved(
629 bool LLVM::DbgValueOp::canUsesBeRemoved(
634 if (blockingUses.size() != 1)
637 return (*blockingUses.begin())->get() == getValue();
649 UndefOp::create(builder, getValue().getLoc(), getValue().
getType());
650 getValueMutable().assign(undef);
654 bool LLVM::DbgDeclareOp::requiresReplacedValues() {
return true; }
656 void LLVM::DbgDeclareOp::visitReplacedValues(
658 for (
auto [op, value] : definitions) {
660 LLVM::DbgValueOp::create(builder, getLoc(), value, getVarInfo(),
670 return llvm::all_of(gepOp.getIndices(), [](
auto index) {
671 auto indexAttr = llvm::dyn_cast_if_present<IntegerAttr>(index);
672 return indexAttr && indexAttr.getValue() == 0;
676 bool LLVM::GEPOp::canUsesBeRemoved(
697 for (
auto index : gep.getIndices()) {
698 auto constIndex = dyn_cast<IntegerAttr>(index);
701 int64_t gepIndex = constIndex.getInt();
705 indices.push_back(gepIndex);
708 Type currentType = gep.getElemType();
709 uint64_t offset = indices[0] * dataLayout.
getTypeSize(currentType);
711 for (uint64_t index : llvm::drop_begin(indices)) {
714 .Case([&](LLVM::LLVMArrayType arrayType) {
716 index * dataLayout.
getTypeSize(arrayType.getElementType());
717 currentType = arrayType.getElementType();
720 .Case([&](LLVM::LLVMStructType structType) {
722 assert(index < body.size() &&
"expected valid struct indexing");
723 for (uint32_t i : llvm::seq(index)) {
724 if (!structType.isPacked())
725 offset = llvm::alignTo(
731 if (!structType.isPacked())
732 offset = llvm::alignTo(
734 currentType = body[index];
737 .Default([&](
Type type) {
738 LDBG() <<
"[sroa] Unsupported type for offset computations"
753 struct SubslotAccessInfo {
757 uint64_t subslotOffset;
765 static std::optional<SubslotAccessInfo>
767 const DataLayout &dataLayout, LLVM::GEPOp gep) {
775 auto isOutOfBoundsGEPIndex = [](uint64_t index) {
783 .Case([&](LLVM::LLVMArrayType arrayType)
784 -> std::optional<SubslotAccessInfo> {
786 uint64_t elemSize = dataLayout.
getTypeSize(arrayType.getElementType());
787 uint64_t index = *offset / elemSize;
788 if (isOutOfBoundsGEPIndex(index))
790 return SubslotAccessInfo{
static_cast<uint32_t
>(index),
791 *offset - (index * elemSize)};
793 .Case([&](LLVM::LLVMStructType structType)
794 -> std::optional<SubslotAccessInfo> {
795 uint64_t distanceToStart = 0;
800 if (!structType.isPacked()) {
801 distanceToStart = llvm::alignTo(
804 if (offset < distanceToStart)
808 if (offset < distanceToStart + elemSize) {
809 if (isOutOfBoundsGEPIndex(index))
813 return SubslotAccessInfo{
static_cast<uint32_t
>(index),
814 *offset - distanceToStart};
819 distanceToStart += elemSize;
833 LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses(
838 std::optional<uint64_t> gepOffset =
gepToByteOffset(dataLayout, *
this);
843 if (*gepOffset >= slotSize)
861 std::optional<SubslotAccessInfo> accessInfo =
868 usedIndices.insert(indexAttr);
873 uint64_t slotSize = dataLayout.
getTypeSize(subslotType);
874 LLVM::LLVMArrayType remainingSlotType =
876 mustBeSafelyUsed.emplace_back<
MemorySlot>({getRes(), remainingSlotType});
885 std::optional<SubslotAccessInfo> accessInfo =
887 assert(accessInfo &&
"expected access info to be checked before");
890 const MemorySlot &newSlot = subslots.at(indexAttr);
894 getLoc(), getResult().getType(), byteType, newSlot.
ptr,
896 getResult().replaceAllUsesWith(newPtr);
908 template <
class MemIntr>
909 std::optional<uint64_t> getStaticMemIntrLen(MemIntr op) {
913 if (memIntrLen.getBitWidth() > 64)
915 return memIntrLen.getZExtValue();
923 std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemcpyInlineOp op) {
924 APInt memIntrLen = op.getLen();
925 if (memIntrLen.getBitWidth() > 64)
927 return memIntrLen.getZExtValue();
935 std::optional<uint64_t> getStaticMemIntrLen(LLVM::MemsetInlineOp op) {
936 APInt memIntrLen = op.getLen();
937 if (memIntrLen.getBitWidth() > 64)
939 return memIntrLen.getZExtValue();
943 template <
class MemsetIntr>
944 IntegerAttr createMemsetLenAttr(MemsetIntr op) {
945 IntegerAttr memsetLenAttr;
946 bool successfulMatch =
947 matchPattern(op.getLen(), m_Constant<IntegerAttr>(&memsetLenAttr));
948 (void)successfulMatch;
949 assert(successfulMatch);
950 return memsetLenAttr;
957 IntegerAttr createMemsetLenAttr(LLVM::MemsetInlineOp op) {
958 return op.getLenAttr();
964 template <
class MemsetIntr>
965 void createMemsetIntr(
OpBuilder &builder, MemsetIntr toReplace,
966 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
971 void createMemsetIntr(
OpBuilder &builder, LLVM::MemsetOp toReplace,
972 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
975 Value newMemsetSizeValue =
976 LLVM::ConstantOp::create(
977 builder, toReplace.getLen().getLoc(),
981 LLVM::MemsetOp::create(builder, toReplace.getLoc(), subslots.at(index).ptr,
982 toReplace.getVal(), newMemsetSizeValue,
983 toReplace.getIsVolatile());
987 void createMemsetIntr(
OpBuilder &builder, LLVM::MemsetInlineOp toReplace,
988 IntegerAttr memsetLenAttr, uint64_t newMemsetSize,
991 auto newMemsetSizeValue =
994 LLVM::MemsetInlineOp::create(builder, toReplace.getLoc(),
995 subslots.at(index).ptr, toReplace.getVal(),
996 newMemsetSizeValue, toReplace.getIsVolatile());
1003 template <
class MemIntr>
1006 if (!isa<LLVM::LLVMPointerType>(slot.
ptr.
getType()) ||
1007 op.getDst() != slot.
ptr)
1010 std::optional<uint64_t> memIntrLen = getStaticMemIntrLen(op);
1020 auto intIndex = dyn_cast<IntegerAttr>(index);
1021 return intIndex && intIndex.getType() == i32;
1029 template <
class MemsetIntr>
1037 if (op.getIsVolatile())
1040 if (!cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap())
1049 template <
class MemsetIntr>
1054 auto buildMemsetValue = [&](
unsigned width) ->
Value {
1055 assert(width % 8 == 0);
1060 IntegerAttr constantPattern;
1062 assert(constantPattern.getValue().getBitWidth() == 8);
1063 APInt memsetVal(width, 0);
1064 for (
unsigned loBit = 0; loBit < width; loBit += 8)
1065 memsetVal.insertBits(constantPattern.getValue(), loBit);
1066 return LLVM::ConstantOp::create(builder, op.getLoc(),
1076 uint64_t coveredBits = 8;
1077 Value currentValue =
1078 LLVM::ZExtOp::create(builder, op.getLoc(), intType, op.getVal());
1079 while (coveredBits < width) {
1081 LLVM::ConstantOp::create(builder, op.getLoc(), intType, coveredBits);
1083 LLVM::ShlOp::create(builder, op.getLoc(), currentValue, shiftBy);
1085 LLVM::OrOp::create(builder, op.getLoc(), currentValue, shifted);
1089 return currentValue;
1092 .Case([&](IntegerType type) ->
Value {
1093 return buildMemsetValue(type.getWidth());
1095 .Case([&](FloatType type) ->
Value {
1096 Value intVal = buildMemsetValue(type.getWidth());
1097 return LLVM::BitcastOp::create(builder, op.getLoc(), type, intVal);
1101 "getStored should not be called on memset to unsupported type");
1105 template <
class MemsetIntr>
1111 bool canConvertType =
1113 .Case<IntegerType, FloatType>([](
auto type) {
1114 return type.getWidth() % 8 == 0 && type.getWidth() > 0;
1116 .Default([](
Type) {
return false; });
1117 if (!canConvertType)
1120 if (op.getIsVolatile())
1126 template <
class MemsetIntr>
1132 std::optional<DenseMap<Attribute, Type>> types =
1133 cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap();
1135 IntegerAttr memsetLenAttr = createMemsetLenAttr(op);
1137 bool packed =
false;
1138 if (
auto structType = dyn_cast<LLVM::LLVMStructType>(slot.
elemType))
1139 packed = structType.isPacked();
1142 uint64_t memsetLen = memsetLenAttr.getValue().getZExtValue();
1143 uint64_t covered = 0;
1144 for (
size_t i = 0; i < types->size(); i++) {
1147 Type elemType = types->at(index);
1148 uint64_t typeSize = dataLayout.
getTypeSize(elemType);
1154 if (covered >= memsetLen)
1159 if (subslots.contains(index)) {
1160 uint64_t newMemsetSize =
std::min(memsetLen - covered, typeSize);
1161 createMemsetIntr(builder, op, memsetLenAttr, newMemsetSize, subslots,
1165 covered += typeSize;
1171 bool LLVM::MemsetOp::loadsFrom(
const MemorySlot &slot) {
return false; }
1173 bool LLVM::MemsetOp::storesTo(
const MemorySlot &slot) {
1174 return getDst() == slot.
ptr;
1183 bool LLVM::MemsetOp::canUsesBeRemoved(
1198 LogicalResult LLVM::MemsetOp::ensureOnlySafeAccesses(
1216 return memsetRewire(*
this, slot, subslots, builder, dataLayout);
1219 bool LLVM::MemsetInlineOp::loadsFrom(
const MemorySlot &slot) {
return false; }
1221 bool LLVM::MemsetInlineOp::storesTo(
const MemorySlot &slot) {
1222 return getDst() == slot.
ptr;
1231 bool LLVM::MemsetInlineOp::canUsesBeRemoved(
1246 LogicalResult LLVM::MemsetInlineOp::ensureOnlySafeAccesses(
1252 bool LLVM::MemsetInlineOp::canRewire(
1265 return memsetRewire(*
this, slot, subslots, builder, dataLayout);
1272 template <
class MemcpyLike>
1274 return op.getSrc() == slot.
ptr;
1277 template <
class MemcpyLike>
1279 return op.getDst() == slot.
ptr;
1282 template <
class MemcpyLike>
1285 return LLVM::LoadOp::create(builder, op.getLoc(), slot.
elemType, op.getSrc());
1288 template <
class MemcpyLike>
1297 if (op.getDst() == op.getSrc())
1300 if (op.getIsVolatile())
1306 template <
class MemcpyLike>
1311 if (op.loadsFrom(slot))
1312 LLVM::StoreOp::create(builder, op.getLoc(), reachingDefinition,
1317 template <
class MemcpyLike>
1318 static LogicalResult
1328 template <
class MemcpyLike>
1333 if (op.getIsVolatile())
1336 if (!cast<DestructurableTypeInterface>(slot.
elemType).getSubelementIndexMap())
1346 if (op.getSrc() == slot.
ptr)
1347 usedIndices.insert_range(llvm::make_first_range(slot.
subelementTypes));
1354 template <
class MemcpyLike>
1357 Type toCpy,
bool isVolatile) {
1359 LLVM::ConstantOp::create(builder, toReplace.getLoc(),
1362 MemcpyLike::create(builder, toReplace.getLoc(), dst, src, memcpySize,
1368 LLVM::MemcpyInlineOp toReplace,
Value dst,
1369 Value src,
Type toCpy,
bool isVolatile) {
1371 toReplace.getLen().getBitWidth());
1372 LLVM::MemcpyInlineOp::create(
1373 builder, toReplace.getLoc(), dst, src,
1381 template <
class MemcpyLike>
1386 if (subslots.empty())
1389 assert((slot.
ptr == op.getDst()) != (slot.
ptr == op.getSrc()));
1390 bool isDst = slot.
ptr == op.getDst();
1393 size_t slotsTreated = 0;
1398 Type indexType = cast<IntegerAttr>(subslots.begin()->first).getType();
1401 if (!subslots.contains(index))
1403 const MemorySlot &subslot = subslots.at(index);
1412 0,
static_cast<int32_t
>(
1413 cast<IntegerAttr>(index).getValue().getZExtValue())};
1414 Value subslotPtrInOther = LLVM::GEPOp::create(
1416 slot.
elemType, isDst ? op.getSrc() : op.getDst(), gepIndices);
1419 createMemcpyLikeToReplace(builder, dataLayout, op,
1420 isDst ? subslot.
ptr : subslotPtrInOther,
1421 isDst ? subslotPtrInOther : subslot.
ptr,
1422 subslot.
elemType, op.getIsVolatile());
1425 assert(subslots.size() == slotsTreated);
1430 bool LLVM::MemcpyOp::loadsFrom(
const MemorySlot &slot) {
1434 bool LLVM::MemcpyOp::storesTo(
const MemorySlot &slot) {
1444 bool LLVM::MemcpyOp::canUsesBeRemoved(
1457 reachingDefinition);
1460 LogicalResult LLVM::MemcpyOp::ensureOnlySafeAccesses(
1478 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1481 bool LLVM::MemcpyInlineOp::loadsFrom(
const MemorySlot &slot) {
1485 bool LLVM::MemcpyInlineOp::storesTo(
const MemorySlot &slot) {
1495 bool LLVM::MemcpyInlineOp::canUsesBeRemoved(
1508 reachingDefinition);
1511 LogicalResult LLVM::MemcpyInlineOp::ensureOnlySafeAccesses(
1517 bool LLVM::MemcpyInlineOp::canRewire(
1530 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1533 bool LLVM::MemmoveOp::loadsFrom(
const MemorySlot &slot) {
1537 bool LLVM::MemmoveOp::storesTo(
const MemorySlot &slot) {
1547 bool LLVM::MemmoveOp::canUsesBeRemoved(
1560 reachingDefinition);
1563 LogicalResult LLVM::MemmoveOp::ensureOnlySafeAccesses(
1581 return memcpyRewire(*
this, slot, subslots, builder, dataLayout);
1588 std::optional<DenseMap<Attribute, Type>>
1589 LLVM::LLVMStructType::getSubelementIndexMap()
const {
1594 return destructured;
1598 auto indexAttr = llvm::dyn_cast<IntegerAttr>(index);
1599 if (!indexAttr || !indexAttr.getType().isInteger(32))
1601 int32_t indexInt = indexAttr.getInt();
1603 if (indexInt < 0 || body.size() <=
static_cast<uint32_t
>(indexInt))
1605 return body[indexInt];
1608 std::optional<DenseMap<Attribute, Type>>
1609 LLVM::LLVMArrayType::getSubelementIndexMap()
const {
1610 constexpr
size_t maxArraySizeForDestructuring = 16;
1617 for (int32_t index = 0; index < numElements; ++index)
1618 destructured.insert({IntegerAttr::get(i32, index), getElementType()});
1619 return destructured;
1623 auto indexAttr = llvm::dyn_cast<IntegerAttr>(index);
1624 if (!indexAttr || !indexAttr.getType().isInteger(32))
1626 int32_t indexInt = indexAttr.getInt();
1627 if (indexInt < 0 ||
getNumElements() <=
static_cast<uint32_t
>(indexInt))
static Value getBase(Value v)
Looks through known "view-like" ops to find the base memref.
static MLIRContext * getContext(OpFoldResult val)
static Type getElementType(Type type)
Determine the element type of type.
static int64_t getNumElements(Type t)
Compute the total number of elements in the given type, also taking into account nested types.
static LLVM::LLVMArrayType getByteArrayType(MLIRContext *context, unsigned size)
Constructs a byte array type of the given size.
static LogicalResult memcpyEnsureOnlySafeAccesses(MemcpyLike op, const MemorySlot &slot, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed)
static std::optional< uint64_t > gepToByteOffset(const DataLayout &dataLayout, LLVM::GEPOp gep)
Returns the amount of bytes the provided GEP elements will offset the pointer by.
static bool areAllIndicesI32(const DestructurableMemorySlot &slot)
Checks whether all indices are i32.
static Value castToSameSizedInt(OpBuilder &builder, Location loc, Value val, const DataLayout &dataLayout)
Converts a value to an integer type of the same size.
static Value castSameSizedTypes(OpBuilder &builder, Location loc, Value srcValue, Type targetType, const DataLayout &dataLayout)
Constructs operations that convert srcValue into a new value of type targetType.
static std::optional< SubslotAccessInfo > getSubslotAccessInfo(const DestructurableMemorySlot &slot, const DataLayout &dataLayout, LLVM::GEPOp gep)
Computes subslot access information for an access into slot with the given offset.
static bool memcpyStoresTo(MemcpyLike op, const MemorySlot &slot)
static DeletionKind memsetRewire(MemsetIntr op, const DestructurableMemorySlot &slot, DenseMap< Attribute, MemorySlot > &subslots, OpBuilder &builder, const DataLayout &dataLayout)
static Type getTypeAtIndex(const DestructurableMemorySlot &slot, Attribute index)
Returns the subslot's type at the requested index.
static bool areConversionCompatible(const DataLayout &layout, Type targetType, Type srcType, bool narrowingConversion)
Checks that rhs can be converted to lhs by a sequence of casts and truncations.
static bool forwardToUsers(Operation *op, SmallVectorImpl< OpOperand * > &newBlockingUses)
Conditions the deletion of the operation to the removal of all its uses.
static bool memsetCanUsesBeRemoved(MemsetIntr op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, SmallVectorImpl< OpOperand * > &newBlockingUses, const DataLayout &dataLayout)
static bool memcpyLoadsFrom(MemcpyLike op, const MemorySlot &slot)
static bool isSupportedTypeForConversion(Type type)
Checks if type can be used in any kind of conversion sequences.
static Value createExtractAndCast(OpBuilder &builder, Location loc, Value srcValue, Type targetType, const DataLayout &dataLayout)
Constructs operations that convert srcValue into a new value of type targetType.
static Value createInsertAndCast(OpBuilder &builder, Location loc, Value srcValue, Value reachingDef, const DataLayout &dataLayout)
Constructs operations that insert the bits of srcValue into the "beginning" of reachingDef (beginning...
static DeletionKind memcpyRemoveBlockingUses(MemcpyLike op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, OpBuilder &builder, Value reachingDefinition)
static bool memcpyCanUsesBeRemoved(MemcpyLike op, const MemorySlot &slot, const SmallPtrSetImpl< OpOperand * > &blockingUses, SmallVectorImpl< OpOperand * > &newBlockingUses, const DataLayout &dataLayout)
static bool isBigEndian(const DataLayout &dataLayout)
Checks if dataLayout describes a little endian layout.
static bool hasAllZeroIndices(LLVM::GEPOp gepOp)
static bool isValidAccessType(const MemorySlot &slot, Type accessType, const DataLayout &dataLayout)
Checks if slot can be accessed through the provided access type.
static Value memcpyGetStored(MemcpyLike op, const MemorySlot &slot, OpBuilder &builder)
static Value castIntValueToSameSizedType(OpBuilder &builder, Location loc, Value val, Type targetType)
Converts a value with an integer type to targetType.
static bool memsetCanRewire(MemsetIntr op, const DestructurableMemorySlot &slot, SmallPtrSetImpl< Attribute > &usedIndices, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed, const DataLayout &dataLayout)
static DeletionKind memcpyRewire(MemcpyLike op, const DestructurableMemorySlot &slot, DenseMap< Attribute, MemorySlot > &subslots, OpBuilder &builder, const DataLayout &dataLayout)
Rewires a memcpy-like operation.
static Value memsetGetStored(MemsetIntr op, const MemorySlot &slot, OpBuilder &builder)
static bool definitelyWritesOnlyWithinSlot(MemIntr op, const MemorySlot &slot, const DataLayout &dataLayout)
Returns whether one can be sure the memory intrinsic does not write outside of the bounds of the give...
static bool memcpyCanRewire(MemcpyLike op, const DestructurableMemorySlot &slot, SmallPtrSetImpl< Attribute > &usedIndices, SmallVectorImpl< MemorySlot > &mustBeSafelyUsed, const DataLayout &dataLayout)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
Attributes are known-constant values of operations.
This class represents an argument of a Block.
IntegerAttr getIntegerAttr(Type type, int64_t value)
IntegerType getIntegerType(unsigned width)
MLIRContext * getContext() const
The main mechanism for performing data layout queries.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Attribute getEndianness() const
Returns the specified endianness.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
MLIRContext is the top-level object for a collection of MLIR operations.
This class helps build Operations.
void setInsertionPoint(Block *block, Block::iterator insertPoint)
Set the insertion point to the specified location.
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
result_range getResults()
void erase()
Remove this operation from its parent block and delete it.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Dialect & getDialect() const
Get the dialect this type is registered to.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
bool use_empty() const
Returns true if this value has no uses.
MLIRContext * getContext() const
Utility to get the associated MLIRContext that this value is defined in.
Type getType() const
Return the type of this value.
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
constexpr int kGEPConstantBitWidth
Bit-width of a 'GEPConstantIndex' within GEPArg.
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
detail::constant_int_value_binder m_ConstantInt(IntegerAttr::ValueType *bind_value)
Matches a constant holding a scalar/vector/tensor integer (splat) and writes the integer value to bin...
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
DeletionKind
Returned by operation promotion logic requesting the deletion of an operation.
@ Keep
Keep the operation after promotion.
@ Delete
Delete the operation after promotion.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Memory slot attached with information about its destructuring procedure.
DenseMap< Attribute, Type > subelementTypes
Maps an index within the memory slot to the corresponding subelement type.
Represents a slot in memory.
Value ptr
Pointer to the memory slot, used by operations to refer to it.
Type elemType
Type of the value contained in the slot.