17 #define GEN_PASS_DEF_SROA
18 #include "mlir/Transforms/Passes.h.inc"
21 #define DEBUG_TYPE "sroa"
30 struct MemorySlotDestructuringInfo {
46 static std::optional<MemorySlotDestructuringInfo>
49 assert(isa<DestructurableTypeInterface>(slot.
elemType));
54 MemorySlotDestructuringInfo info;
58 auto scheduleAsBlockingUse = [&](
OpOperand &use) {
60 info.userToBlockingUses.getOrInsertDefault(use.getOwner());
61 blockingUses.insert(&use);
67 dyn_cast<DestructurableAccessorOpInterface>(use.getOwner())) {
68 if (accessor.canRewire(slot, info.usedIndices, usedSafelyWorklist,
70 info.accessors.push_back(accessor);
77 scheduleAsBlockingUse(use);
81 while (!usedSafelyWorklist.empty()) {
82 MemorySlot mustBeUsedSafely = usedSafelyWorklist.pop_back_val();
84 if (!visited.insert(&subslotUse).second)
86 Operation *subslotUser = subslotUse.getOwner();
88 if (
auto memOp = dyn_cast<SafeMemorySlotAccessOpInterface>(subslotUser))
89 if (succeeded(memOp.ensureOnlySafeAccesses(
90 mustBeUsedSafely, usedSafelyWorklist, dataLayout)))
95 scheduleAsBlockingUse(subslotUse);
103 if (!info.userToBlockingUses.contains(user))
107 auto promotable = dyn_cast<PromotableOpInterface>(user);
117 if (!promotable.canUsesBeRemoved(blockingUses, newBlockingUses, dataLayout))
121 for (
OpOperand *blockingUse : newBlockingUses) {
122 assert(llvm::is_contained(user->getResults(), blockingUse->get()));
125 info.userToBlockingUses.getOrInsertDefault(blockingUse->getOwner());
126 newUserBlockingUseSet.insert(blockingUse);
138 DestructurableAllocationOpInterface allocator,
OpBuilder &builder,
139 const DataLayout &dataLayout, MemorySlotDestructuringInfo &info,
146 allocator.destructure(slot, info.usedIndices, builder, newAllocators);
156 for (
Operation *user : llvm::make_first_range(info.userToBlockingUses))
157 usersToRewire.insert(user);
158 for (DestructurableAccessorOpInterface accessor : info.accessors)
159 usersToRewire.insert(accessor);
163 for (
Operation *toRewire : llvm::reverse(usersToRewire)) {
165 if (
auto accessor = dyn_cast<DestructurableAccessorOpInterface>(toRewire)) {
166 if (accessor.rewire(slot, subslots, builder, dataLayout) ==
168 toErase.push_back(accessor);
172 auto promotable = cast<PromotableOpInterface>(toRewire);
173 if (promotable.removeBlockingUses(info.userToBlockingUses[promotable],
175 toErase.push_back(promotable);
181 assert(slot.
ptr.
use_empty() &&
"after destructuring, the original slot "
182 "pointer should no longer be used");
184 LLVM_DEBUG(llvm::dbgs() <<
"[sroa] Destructured memory slot: " << slot.
ptr
190 std::optional<DestructurableAllocationOpInterface> newAllocator =
191 allocator.handleDestructuringComplete(slot, builder);
194 newAllocators.push_back(*newAllocator);
201 bool destructuredAny =
false;
206 newWorkList.reserve(allocators.size());
210 bool changesInThisRound =
false;
212 for (DestructurableAllocationOpInterface allocator : workList) {
213 bool destructuredAnySlot =
false;
215 std::optional<MemorySlotDestructuringInfo> info =
221 newWorkList, statistics);
222 destructuredAnySlot =
true;
228 if (!destructuredAnySlot)
229 newWorkList.push_back(allocator);
230 changesInThisRound |= destructuredAnySlot;
233 if (!changesInThisRound)
235 destructuredAny |= changesInThisRound;
239 workList.swap(newWorkList);
243 return success(destructuredAny);
248 struct SROA :
public impl::SROABase<SROA> {
249 using impl::SROABase<SROA>::SROABase;
251 void runOnOperation()
override {
254 SROAStatistics statistics{&destructuredAmount, &slotsWithMemoryBenefit,
255 &maxSubelementAmount};
257 auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
258 const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(scopeOp);
259 bool changed =
false;
262 if (region.getBlocks().empty())
265 OpBuilder builder(®ion.front(), region.front().begin());
269 region.walk([&](DestructurableAllocationOpInterface allocator) {
270 allocators.emplace_back(allocator);
279 markAllAnalysesPreserved();
static void destructureSlot(DestructurableMemorySlot &slot, DestructurableAllocationOpInterface allocator, OpBuilder &builder, const DataLayout &dataLayout, MemorySlotDestructuringInfo &info, SmallVectorImpl< DestructurableAllocationOpInterface > &newAllocators, const SROAStatistics &statistics)
Performs the destructuring of a destructible slot given associated destructuring information.
static std::optional< MemorySlotDestructuringInfo > computeDestructuringInfo(DestructurableMemorySlot &slot, const DataLayout &dataLayout)
Computes information for slot destructuring.
The main mechanism for performing data layout queries.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
bool use_empty() const
Returns true if this value has no uses.
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Block * getParentBlock()
Return the Block in which this Value is defined.
Include the generated interface declarations.
@ Delete
Delete the operation after promotion.
LogicalResult tryToDestructureMemorySlots(ArrayRef< DestructurableAllocationOpInterface > allocators, OpBuilder &builder, const DataLayout &dataLayout, SROAStatistics statistics={})
Attempts to destructure the slots of destructurable allocators.
SetVector< Operation * > topologicalSort(const SetVector< Operation * > &toSort)
Sorts all operations in toSort topologically while also considering region semantics.
void getForwardSlice(Operation *op, SetVector< Operation * > *forwardSlice, const ForwardSliceOptions &options={})
Fills forwardSlice with the computed forward slice (i.e.
Memory slot attached with information about its destructuring procedure.
DenseMap< Attribute, Type > subelementTypes
Maps an index within the memory slot to the corresponding subelement type.
Represents a slot in memory.
Value ptr
Pointer to the memory slot, used by operations to refer to it.
Type elemType
Type of the value contained in the slot.
Statistics collected while applying SROA.
llvm::Statistic * maxSubelementAmount
Maximal number of sub-elements a successfully destructured slot initially had.
llvm::Statistic * slotsWithMemoryBenefit
Total amount of memory slots in which the destructured size was smaller than the total size after eli...
llvm::Statistic * destructuredAmount
Total amount of memory slots destructured.