16 #define GEN_PASS_DEF_SROA
17 #include "mlir/Transforms/Passes.h.inc"
20 #define DEBUG_TYPE "sroa"
29 struct MemorySlotDestructuringInfo {
45 static std::optional<MemorySlotDestructuringInfo>
48 assert(isa<DestructurableTypeInterface>(slot.
elemType));
53 MemorySlotDestructuringInfo info;
57 auto scheduleAsBlockingUse = [&](
OpOperand &use) {
59 info.userToBlockingUses.getOrInsertDefault(use.getOwner());
60 blockingUses.insert(&use);
66 dyn_cast<DestructurableAccessorOpInterface>(use.getOwner())) {
67 if (accessor.canRewire(slot, info.usedIndices, usedSafelyWorklist,
69 info.accessors.push_back(accessor);
76 scheduleAsBlockingUse(use);
80 while (!usedSafelyWorklist.empty()) {
81 MemorySlot mustBeUsedSafely = usedSafelyWorklist.pop_back_val();
83 if (!visited.insert(&subslotUse).second)
85 Operation *subslotUser = subslotUse.getOwner();
87 if (
auto memOp = dyn_cast<SafeMemorySlotAccessOpInterface>(subslotUser))
88 if (
succeeded(memOp.ensureOnlySafeAccesses(
89 mustBeUsedSafely, usedSafelyWorklist, dataLayout)))
94 scheduleAsBlockingUse(subslotUse);
102 if (!info.userToBlockingUses.contains(user))
106 auto promotable = dyn_cast<PromotableOpInterface>(user);
116 if (!promotable.canUsesBeRemoved(blockingUses, newBlockingUses, dataLayout))
120 for (
OpOperand *blockingUse : newBlockingUses) {
121 assert(llvm::is_contained(user->getResults(), blockingUse->get()));
124 info.userToBlockingUses.getOrInsertDefault(blockingUse->getOwner());
125 newUserBlockingUseSet.insert(blockingUse);
136 DestructurableAllocationOpInterface allocator,
139 MemorySlotDestructuringInfo &info,
141 RewriterBase::InsertionGuard guard(rewriter);
145 allocator.destructure(slot, info.usedIndices, rewriter);
148 slot.
elementPtrs.size() != info.usedIndices.size())
155 for (
Operation *user : llvm::make_first_range(info.userToBlockingUses))
156 usersToRewire.insert(user);
157 for (DestructurableAccessorOpInterface accessor : info.accessors)
158 usersToRewire.insert(accessor);
162 for (
Operation *toRewire : llvm::reverse(usersToRewire)) {
164 if (
auto accessor = dyn_cast<DestructurableAccessorOpInterface>(toRewire)) {
165 if (accessor.rewire(slot, subslots, rewriter, dataLayout) ==
167 toErase.push_back(accessor);
171 auto promotable = cast<PromotableOpInterface>(toRewire);
172 if (promotable.removeBlockingUses(info.userToBlockingUses[promotable],
174 toErase.push_back(promotable);
180 assert(slot.
ptr.
use_empty() &&
"after destructuring, the original slot "
181 "pointer should no longer be used");
183 LLVM_DEBUG(llvm::dbgs() <<
"[sroa] Destructured memory slot: " << slot.
ptr
189 allocator.handleDestructuringComplete(slot, rewriter);
196 bool destructuredAny =
false;
198 for (DestructurableAllocationOpInterface allocator : allocators) {
200 std::optional<MemorySlotDestructuringInfo> info =
205 destructureSlot(slot, allocator, rewriter, dataLayout, *info, statistics);
206 destructuredAny =
true;
210 return success(destructuredAny);
215 struct SROA :
public impl::SROABase<SROA> {
216 using impl::SROABase<SROA>::SROABase;
218 void runOnOperation()
override {
221 SROAStatistics statistics{&destructuredAmount, &slotsWithMemoryBenefit,
222 &maxSubelementAmount};
224 auto &dataLayoutAnalysis = getAnalysis<DataLayoutAnalysis>();
225 const DataLayout &dataLayout = dataLayoutAnalysis.getAtOrAbove(scopeOp);
226 bool changed =
false;
229 if (region.getBlocks().empty())
232 OpBuilder builder(®ion.front(), region.front().begin());
242 region.walk([&](DestructurableAllocationOpInterface allocator) {
243 allocators.emplace_back(allocator);
254 markAllAnalysesPreserved();
static void destructureSlot(DestructurableMemorySlot &slot, DestructurableAllocationOpInterface allocator, RewriterBase &rewriter, const DataLayout &dataLayout, MemorySlotDestructuringInfo &info, const SROAStatistics &statistics)
Performs the destructuring of a destructible slot given associated destructuring information.
static std::optional< MemorySlotDestructuringInfo > computeDestructuringInfo(DestructurableMemorySlot &slot, const DataLayout &dataLayout)
Computes information for slot destructuring.
The main mechanism for performing data layout queries.
This class coordinates rewriting a piece of IR outside of a pattern rewrite, providing a way to keep ...
This class helps build Operations.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
This class contains a list of basic blocks and a link to the parent operation it is attached to.
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
bool use_empty() const
Returns true if this value has no uses.
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Block * getParentBlock()
Return the Block in which this Value is defined.
Include the generated interface declarations.
LogicalResult tryToDestructureMemorySlots(ArrayRef< DestructurableAllocationOpInterface > allocators, RewriterBase &rewriter, const DataLayout &dataLayout, SROAStatistics statistics={})
Attempts to destructure the slots of destructurable allocators.
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
@ Delete
Delete the operation after promotion.
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
SetVector< Operation * > topologicalSort(const SetVector< Operation * > &toSort)
Multi-root DAG topological sort.
void getForwardSlice(Operation *op, SetVector< Operation * > *forwardSlice, const ForwardSliceOptions &options={})
Fills forwardSlice with the computed forward slice (i.e.
Memory slot attached with information about its destructuring procedure.
DenseMap< Attribute, Type > elementPtrs
Maps an index within the memory slot to the corresponding subelement type.
This class represents an efficient way to signal success or failure.
Represents a slot in memory.
Value ptr
Pointer to the memory slot, used by operations to refer to it.
Type elemType
Type of the value contained in the slot.
Statistics collected while applying SROA.
llvm::Statistic * maxSubelementAmount
Maximal number of sub-elements a successfully destructured slot initially had.
llvm::Statistic * slotsWithMemoryBenefit
Total amount of memory slots in which the destructured size was smaller than the total size after eli...
llvm::Statistic * destructuredAmount
Total amount of memory slots destructured.