26 #include "llvm/ADT/TypeSwitch.h"
27 #include "llvm/Support/Debug.h"
28 #include "llvm/Support/DebugLog.h"
31 #define GEN_PASS_DEF_GPUELIMINATEBARRIERS
32 #include "mlir/Dialect/GPU/Transforms/Passes.h.inc"
38 #define DEBUG_TYPE "gpu-erase-barriers"
39 #define DEBUG_TYPE_ALIAS "gpu-erase-barries-alias"
47 if (op->
hasAttr(
"__parallel_region_boundary_for_test"))
50 return isa<GPUFuncOp, LaunchOp>(op);
61 return isa<FunctionOpInterface, scf::IfOp, memref::AllocaScopeOp>(op);
68 return isa_and_nonnull<memref::AllocOp, memref::AllocaOp>(op);
75 effects.emplace_back(MemoryEffects::Effect::get<MemoryEffects::Read>());
76 effects.emplace_back(MemoryEffects::Effect::get<MemoryEffects::Write>());
77 effects.emplace_back(MemoryEffects::Effect::get<MemoryEffects::Allocate>());
78 effects.emplace_back(MemoryEffects::Effect::get<MemoryEffects::Free>());
88 bool ignoreBarriers =
true) {
91 if (ignoreBarriers && isa<BarrierOp>(op))
98 if (
auto iface = dyn_cast<MemoryEffectOpInterface>(op)) {
100 iface.getEffects(localEffects);
101 llvm::append_range(effects, localEffects);
106 for (
auto &block : region) {
107 for (
auto &innerOp : block)
126 bool stopAtBarrier) {
130 for (
Operation *it = op->getPrevNode(); it !=
nullptr;
131 it = it->getPrevNode()) {
132 if (isa<BarrierOp>(it)) {
153 bool stopAtBarrier) {
199 bool conservative =
false;
211 return !conservative;
219 bool stopAtBarrier) {
223 for (
Operation *it = op->getNextNode(); it !=
nullptr;
224 it = it->getNextNode()) {
225 if (isa<BarrierOp>(it)) {
245 bool stopAtBarrier) {
296 bool conservative =
false;
308 return !conservative;
318 bool shouldContinue =
320 .Case<memref::CastOp, memref::SubViewOp, memref::ViewOp>(
325 .Case<memref::TransposeOp>([&](
auto op) {
329 .Case<memref::CollapseShapeOp, memref::ExpandShapeOp>([&](
auto op) {
333 .Default([](
Operation *) {
return false; });
342 auto arg = dyn_cast<BlockArgument>(v);
343 return arg && isa<FunctionOpInterface>(arg.getOwner()->getParentOp());
352 [](ViewLikeOpInterface viewLike) {
return viewLike.getViewSource(); })
353 .Case([](CastOpInterface castLike) {
return castLike->getOperand(0); })
354 .Case([](memref::TransposeOp transpose) {
return transpose.getIn(); })
355 .Case<memref::ExpandShapeOp, memref::CollapseShapeOp>(
356 [](
auto op) {
return op.getSrc(); })
367 .Case<memref::StoreOp, vector::TransferWriteOp>(
368 [&](
auto op) {
return op.getValue() == v; })
369 .Case<vector::StoreOp, vector::MaskedStoreOp>(
370 [&](
auto op) {
return op.getValueToStore() == v; })
372 .Case([](memref::DeallocOp) {
return false; })
374 .Default([](
Operation *) {
return std::nullopt; });
383 while (!todo.empty()) {
384 Value v = todo.pop_back_val();
387 auto iface = dyn_cast<MemoryEffectOpInterface>(user);
390 iface.getEffects(effects);
391 if (llvm::all_of(effects,
393 return isa<MemoryEffects::Read>(effect.
getEffect());
407 if (!knownCaptureStatus || *knownCaptureStatus)
426 <<
"checking aliasing between " << first <<
" and " << second;
435 if (first == second) {
441 if (
auto globFirst = first.
getDefiningOp<memref::GetGlobalOp>()) {
442 if (
auto globSecond = second.
getDefiningOp<memref::GetGlobalOp>()) {
443 return globFirst.getNameAttr() == globSecond.getNameAttr();
448 auto isNoaliasFuncArgument = [](
Value value) {
449 auto bbArg = dyn_cast<BlockArgument>(value);
452 auto iface = dyn_cast<FunctionOpInterface>(bbArg.getOwner()->getParentOp());
456 return iface.getArgAttr(bbArg.getArgNumber(),
"llvm.noalias") !=
nullptr;
458 if (isNoaliasFuncArgument(first) && isNoaliasFuncArgument(second))
463 bool isGlobal[] = {first.
getDefiningOp<memref::GetGlobalOp>() !=
nullptr,
469 if ((isDistinct[0] || isGlobal[0]) && (isDistinct[1] || isGlobal[1]))
475 if ((isDistinct[0] && isArg[1]) || (isDistinct[1] && isArg[0]))
533 if (isa<MemoryEffects::Read>(before.getEffect()) &&
534 isa<MemoryEffects::Read>(after.getEffect())) {
542 if (isa<MemoryEffects::Allocate>(before.getEffect()) ||
543 isa<MemoryEffects::Allocate>(after.getEffect())) {
555 if (isa<MemoryEffects::Free>(before.getEffect()))
559 LDBG() <<
"found a conflict between (before): " << before.getValue()
560 <<
" read:" << isa<MemoryEffects::Read>(before.getEffect())
561 <<
" write:" << isa<MemoryEffects::Write>(before.getEffect())
562 <<
" alloc:" << isa<MemoryEffects::Allocate>(before.getEffect())
563 <<
" free:" << isa<MemoryEffects::Free>(before.getEffect());
564 LDBG() <<
"and (after): " << after.getValue()
565 <<
" read:" << isa<MemoryEffects::Read>(after.getEffect())
566 <<
" write:" << isa<MemoryEffects::Write>(after.getEffect())
567 <<
" alloc:" << isa<MemoryEffects::Allocate>(after.getEffect())
568 <<
" free:" << isa<MemoryEffects::Free>(after.getEffect());
581 LogicalResult matchAndRewrite(BarrierOp barrier,
583 LDBG() <<
"checking the necessity of: " << barrier <<
" "
593 LDBG() <<
"the surrounding barriers are sufficient, removing " << barrier;
598 LDBG() <<
"barrier is necessary: " << barrier <<
" " << barrier.getLoc();
603 class GpuEliminateBarriersPass
604 :
public impl::GpuEliminateBarriersBase<GpuEliminateBarriersPass> {
605 void runOnOperation()
override {
606 auto funcOp = getOperation();
610 return signalPassFailure();
static bool isSequentialLoopLike(Operation *op)
Returns true if the op behaves like a sequential loop, e.g., the control flow "wraps around" from the...
static bool isFunctionArgument(Value v)
Returns true if the value is defined as a function argument.
static Value getBase(Value v)
Looks through known "view-like" ops to find the base memref.
static Value propagatesCapture(Operation *op)
Returns the operand that the operation "propagates" through it for capture purposes.
static bool hasSingleExecutionBody(Operation *op)
Returns true if the regions of the op are guaranteed to be executed at most once.
static bool producesDistinctBase(Operation *op)
Returns true if the operation is known to produce a pointer-like object distinct from any other objec...
static bool mayAlias(Value first, Value second)
Returns true if two values may be referencing aliasing memory.
static bool getEffectsBeforeInBlock(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool stopAtBarrier)
Get all effects before the given operation caused by other operations in the same block.
static bool isParallelRegionBoundary(Operation *op)
Returns true if the op is defines the parallel region that is subject to barrier synchronization.
static bool getEffectsAfter(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool stopAtBarrier)
Collects memory effects from operations that may be executed after op in a trivial structured control...
static std::optional< bool > getKnownCapturingStatus(Operation *op, Value v)
Returns true if the given operation is known to capture the given value, false if it is known not to ...
static bool collectEffects(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool ignoreBarriers=true)
Collect the memory effects of the given op in 'effects'.
static bool haveConflictingEffects(ArrayRef< MemoryEffects::EffectInstance > beforeEffects, ArrayRef< MemoryEffects::EffectInstance > afterEffects)
Returns true if any of the "before" effect instances has a conflict with any "after" instance for the...
static bool getEffectsAfterInBlock(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool stopAtBarrier)
Get all effects after the given operation caused by other operations in the same block.
static void addAllValuelessEffects(SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
Populates effects with all memory effects without associating them to a specific value.
static bool maybeCaptured(Value v)
Returns true if the value may be captured by any of its users, i.e., if the user may be storing this ...
static bool getEffectsBefore(Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool stopAtBarrier)
Collects memory effects from operations that may be executed before op in a trivial structured contro...
static MLIRContext * getContext(OpFoldResult val)
Region * getParent() const
Provide a 'getParent' method for ilist_node_with_parent methods.
Operation * getTerminator()
Get the terminator operation of this block.
This trait indicates that the memory effects of an operation includes the effects of operations neste...
This class provides the API for ops that are known to be isolated from above.
Operation is the basic unit of execution within MLIR.
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
bool hasAttr(StringAttr name)
Return true if the operation has an attribute with the provided name, false otherwise.
std::enable_if_t< llvm::function_traits< std::decay_t< FnT > >::num_args==1, RetT > walk(FnT &&callback)
Walk the operation by calling the callback for each nested operation (including this one),...
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Block * getBlock()
Returns the operation block that contains this operation.
MutableArrayRef< Region > getRegions()
Returns the regions held by this operation.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
This class contains a list of basic blocks and a link to the parent operation it is attached to.
bool hasOneBlock()
Return true if this region has exactly one block.
virtual void eraseOp(Operation *op)
This method erases an operation that is known to have no uses.
This class represents a specific instance of an effect.
Resource * getResource() const
Return the resource that the effect applies to.
EffectT * getEffect() const
Return the effect being applied.
Value getValue() const
Return the value the effect is applied on, or nullptr if there isn't a known value being affected.
TypeID getResourceID() const
Return the unique identifier for the base resource class.
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
user_range getUsers() const
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
static WalkResult advance()
static WalkResult interrupt()
Include the generated interface declarations.
LogicalResult applyPatternsGreedily(Region ®ion, const FrozenRewritePatternSet &patterns, GreedyRewriteConfig config=GreedyRewriteConfig(), bool *changed=nullptr)
Rewrite ops in the given region, which must be isolated from above, by repeatedly applying the highes...
const FrozenRewritePatternSet & patterns
void populateGpuEliminateBarriersPatterns(RewritePatternSet &patterns)
Erase barriers that do not enforce conflicting memory side effects.
OpRewritePattern is a wrapper around RewritePattern that allows for matching and rewriting against an...