17 #define GEN_PASS_DEF_SROA
18 #include "mlir/Transforms/Passes.h.inc"
21 #define DEBUG_TYPE "sroa"
30 struct MemorySlotDestructuringInfo {
46 static std::optional<MemorySlotDestructuringInfo>
49 assert(isa<DestructurableTypeInterface>(slot.
elemType));
54 MemorySlotDestructuringInfo info;
58 auto scheduleAsBlockingUse = [&](
OpOperand &use) {
60 info.userToBlockingUses[use.getOwner()];
61 blockingUses.insert(&use);
67 dyn_cast<DestructurableAccessorOpInterface>(use.getOwner())) {
68 if (accessor.canRewire(slot, info.usedIndices, usedSafelyWorklist,
70 info.accessors.push_back(accessor);
77 scheduleAsBlockingUse(use);
81 while (!usedSafelyWorklist.empty()) {
82 MemorySlot mustBeUsedSafely = usedSafelyWorklist.pop_back_val();
84 if (!visited.insert(&subslotUse).second)
86 Operation *subslotUser = subslotUse.getOwner();
88 if (
auto memOp = dyn_cast<SafeMemorySlotAccessOpInterface>(subslotUser))
89 if (succeeded(memOp.ensureOnlySafeAccesses(
90 mustBeUsedSafely, usedSafelyWorklist, dataLayout)))
95 scheduleAsBlockingUse(subslotUse);
103 auto it = info.userToBlockingUses.find(user);
104 if (it == info.userToBlockingUses.end())
108 auto promotable = dyn_cast<PromotableOpInterface>(user);
118 if (!promotable.canUsesBeRemoved(blockingUses, newBlockingUses, dataLayout))
122 for (
OpOperand *blockingUse : newBlockingUses) {
123 assert(llvm::is_contained(user->getResults(), blockingUse->get()));
126 info.userToBlockingUses[blockingUse->getOwner()];
127 newUserBlockingUseSet.insert(blockingUse);
139 DestructurableAllocationOpInterface allocator,
OpBuilder &builder,
140 const DataLayout &dataLayout, MemorySlotDestructuringInfo &info,
147 allocator.destructure(slot, info.usedIndices, builder, newAllocators);
157 usersToRewire.insert_range(llvm::make_first_range(info.userToBlockingUses));
158 usersToRewire.insert_range(info.accessors);
162 for (
Operation *toRewire : llvm::reverse(usersToRewire)) {
164 if (
auto accessor = dyn_cast<DestructurableAccessorOpInterface>(toRewire)) {
165 if (accessor.rewire(slot, subslots, builder, dataLayout) ==
167 toErase.push_back(accessor);
171 auto promotable = cast<PromotableOpInterface>(toRewire);
172 if (promotable.removeBlockingUses(info.userToBlockingUses[promotable],
174 toErase.push_back(promotable);
180 assert(slot.
ptr.
use_empty() &&
"after destructuring, the original slot "
181 "pointer should no longer be used");
183 LLVM_DEBUG(llvm::dbgs() <<
"[sroa] Destructured memory slot: " << slot.
ptr
189 std::optional<DestructurableAllocationOpInterface> newAllocator =
190 allocator.handleDestructuringComplete(slot, builder);
193 newAllocators.push_back(*newAllocator);
200 bool destructuredAny =
false;
204 newWorkList.reserve(allocators.size());
208 bool changesInThisRound =
false;
210 for (DestructurableAllocationOpInterface allocator : workList) {
211 bool destructuredAnySlot =
false;
213 std::optional<MemorySlotDestructuringInfo> info =
219 newWorkList, statistics);
220 destructuredAnySlot =
true;
226 if (!destructuredAnySlot)
227 newWorkList.push_back(allocator);
228 changesInThisRound |= destructuredAnySlot;
231 if (!changesInThisRound)
233 destructuredAny |= changesInThisRound;
237 workList.swap(newWorkList);
241 return success(destructuredAny);
246 struct SROA :
public impl::SROABase<SROA> {
247 using impl::SROABase<SROA>::SROABase;
249 void runOnOperation()
override {
252 SROAStatistics statistics{&destructuredAmount, &slotsWithMemoryBenefit,
253 &maxSubelementAmount};
255 auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
256 const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(scopeOp);
260 if (region.getBlocks().empty())
263 OpBuilder builder(®ion.front(), region.front().begin());
267 region.walk([&](DestructurableAllocationOpInterface allocator) {
268 allocators.emplace_back(allocator);
277 markAllAnalysesPreserved();
static void destructureSlot(DestructurableMemorySlot &slot, DestructurableAllocationOpInterface allocator, OpBuilder &builder, const DataLayout &dataLayout, MemorySlotDestructuringInfo &info, SmallVectorImpl< DestructurableAllocationOpInterface > &newAllocators, const SROAStatistics &statistics)
Performs the destructuring of a destructible slot given associated destructuring information.
static std::optional< MemorySlotDestructuringInfo > computeDestructuringInfo(DestructurableMemorySlot &slot, const DataLayout &dataLayout)
Computes information for slot destructuring.
The main mechanism for performing data layout queries.
RAII guard to reset the insertion point of the builder when destroyed.
This class helps build Operations.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
bool use_empty() const
Returns true if this value has no uses.
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Block * getParentBlock()
Return the Block in which this Value is defined.
Include the generated interface declarations.
const FrozenRewritePatternSet GreedyRewriteConfig bool * changed
@ Delete
Delete the operation after promotion.
LogicalResult tryToDestructureMemorySlots(ArrayRef< DestructurableAllocationOpInterface > allocators, OpBuilder &builder, const DataLayout &dataLayout, SROAStatistics statistics={})
Attempts to destructure the slots of destructurable allocators.
SetVector< Operation * > topologicalSort(const SetVector< Operation * > &toSort)
Sorts all operations in toSort topologically while also considering region semantics.
void getForwardSlice(Operation *op, SetVector< Operation * > *forwardSlice, const ForwardSliceOptions &options={})
Fills forwardSlice with the computed forward slice (i.e.
Memory slot attached with information about its destructuring procedure.
DenseMap< Attribute, Type > subelementTypes
Maps an index within the memory slot to the corresponding subelement type.
Represents a slot in memory.
Value ptr
Pointer to the memory slot, used by operations to refer to it.
Type elemType
Type of the value contained in the slot.
Statistics collected while applying SROA.
llvm::Statistic * maxSubelementAmount
Maximal number of sub-elements a successfully destructured slot initially had.
llvm::Statistic * slotsWithMemoryBenefit
Total amount of memory slots in which the destructured size was smaller than the total size after eli...
llvm::Statistic * destructuredAmount
Total amount of memory slots destructured.