16 #define GEN_PASS_DEF_SROA
17 #include "mlir/Transforms/Passes.h.inc"
20 #define DEBUG_TYPE "sroa"
29 struct MemorySlotDestructuringInfo {
45 static std::optional<MemorySlotDestructuringInfo>
47 assert(isa<DestructurableTypeInterface>(slot.
elemType));
52 MemorySlotDestructuringInfo info;
56 auto scheduleAsBlockingUse = [&](
OpOperand &use) {
58 info.userToBlockingUses.getOrInsertDefault(use.getOwner());
59 blockingUses.insert(&use);
65 dyn_cast<DestructurableAccessorOpInterface>(use.getOwner())) {
66 if (accessor.canRewire(slot, info.usedIndices, usedSafelyWorklist)) {
67 info.accessors.push_back(accessor);
74 scheduleAsBlockingUse(use);
78 while (!usedSafelyWorklist.empty()) {
79 MemorySlot mustBeUsedSafely = usedSafelyWorklist.pop_back_val();
81 if (!visited.insert(&subslotUse).second)
83 Operation *subslotUser = subslotUse.getOwner();
85 if (
auto memOp = dyn_cast<SafeMemorySlotAccessOpInterface>(subslotUser))
86 if (
succeeded(memOp.ensureOnlySafeAccesses(mustBeUsedSafely,
92 scheduleAsBlockingUse(subslotUse);
100 if (!info.userToBlockingUses.contains(user))
104 auto promotable = dyn_cast<PromotableOpInterface>(user);
114 if (!promotable.canUsesBeRemoved(blockingUses, newBlockingUses))
118 for (
OpOperand *blockingUse : newBlockingUses) {
119 assert(llvm::is_contained(user->getResults(), blockingUse->get()));
122 info.userToBlockingUses.getOrInsertDefault(blockingUse->getOwner());
123 newUserBlockingUseSet.insert(blockingUse);
134 DestructurableAllocationOpInterface allocator,
136 MemorySlotDestructuringInfo &info,
138 RewriterBase::InsertionGuard guard(rewriter);
142 allocator.destructure(slot, info.usedIndices, rewriter);
145 slot.
elementPtrs.size() != info.usedIndices.size())
152 for (
Operation *user : llvm::make_first_range(info.userToBlockingUses))
153 usersToRewire.insert(user);
154 for (DestructurableAccessorOpInterface accessor : info.accessors)
155 usersToRewire.insert(accessor);
159 for (
Operation *toRewire : llvm::reverse(usersToRewire)) {
161 if (
auto accessor = dyn_cast<DestructurableAccessorOpInterface>(toRewire)) {
163 toErase.push_back(accessor);
167 auto promotable = cast<PromotableOpInterface>(toRewire);
168 if (promotable.removeBlockingUses(info.userToBlockingUses[promotable],
170 toErase.push_back(promotable);
176 assert(slot.
ptr.
use_empty() &&
"after destructuring, the original slot "
177 "pointer should no longer be used");
179 LLVM_DEBUG(llvm::dbgs() <<
"[sroa] Destructured memory slot: " << slot.
ptr
185 allocator.handleDestructuringComplete(slot, rewriter);
191 bool destructuredAny =
false;
193 for (DestructurableAllocationOpInterface allocator : allocators) {
195 std::optional<MemorySlotDestructuringInfo> info =
201 destructuredAny =
true;
205 return success(destructuredAny);
217 struct SROA :
public impl::SROABase<SROA> {
218 using impl::SROABase<SROA>::SROABase;
220 void runOnOperation()
override {
223 SROAStatistics statistics{&destructuredAmount, &slotsWithMemoryBenefit,
224 &maxSubelementAmount};
static MLIRContext * getContext(OpFoldResult val)
static std::optional< MemorySlotDestructuringInfo > computeDestructuringInfo(DestructurableMemorySlot &slot)
Computes information for slot destructuring.
static void destructureSlot(DestructurableMemorySlot &slot, DestructurableAllocationOpInterface allocator, RewriterBase &rewriter, MemorySlotDestructuringInfo &info, const SROAStatistics &statistics)
Performs the destructuring of a destructible slot given associated destructuring information.
This class represents a frozen set of patterns that can be processed by a pattern applicator.
void setInsertionPointToStart(Block *block)
Sets the insertion point to the start of the specified block.
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
This class represents an operand of an operation.
Operation is the basic unit of execution within MLIR.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
bool hasBoundedRewriteRecursion() const
Returns true if this pattern is known to result in recursive application, i.e.
This class coordinates the application of a rewrite on a set of IR, providing a way for clients to tr...
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
Pattern applying SROA to the regions of the operations on which it matches.
LogicalResult matchAndRewrite(DestructurableAllocationOpInterface allocator, PatternRewriter &rewriter) const override
bool use_empty() const
Returns true if this value has no uses.
use_range getUses() const
Returns a range of all uses, which is useful for iterating over all uses.
Block * getParentBlock()
Return the Block in which this Value is defined.
Include the generated interface declarations.
bool succeeded(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a success value.
LogicalResult tryToDestructureMemorySlots(ArrayRef< DestructurableAllocationOpInterface > allocators, RewriterBase &rewriter, SROAStatistics statistics={})
Attempts to destructure the slots of destructurable allocators.
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
@ Delete
Delete the operation after promotion.
LogicalResult applyPatternsAndFoldGreedily(Region ®ion, const FrozenRewritePatternSet &patterns, GreedyRewriteConfig config=GreedyRewriteConfig(), bool *changed=nullptr)
Rewrite ops in the given region, which must be isolated from above, by repeatedly applying the highes...
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
SetVector< Operation * > topologicalSort(const SetVector< Operation * > &toSort)
Multi-root DAG topological sort.
void getForwardSlice(Operation *op, SetVector< Operation * > *forwardSlice, const ForwardSliceOptions &options={})
Fills forwardSlice with the computed forward slice (i.e.
Memory slot attached with information about its destructuring procedure.
DenseMap< Attribute, Type > elementPtrs
Maps an index within the memory slot to the type of the pointer that will be generated to access the ...
This class represents an efficient way to signal success or failure.
Represents a slot in memory.
Value ptr
Pointer to the memory slot, used by operations to refer to it.
Type elemType
Type of the value contained in the slot.
Statistics collected while applying SROA.
llvm::Statistic * maxSubelementAmount
Maximal number of sub-elements a successfully destructured slot initially had.
llvm::Statistic * slotsWithMemoryBenefit
Total amount of memory slots in which the destructured size was smaller than the total size after eli...
llvm::Statistic * destructuredAmount
Total amount of memory slots destructured.