#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/Operation.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "llvm/ADT/SetOperations.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Debug.h"
#include "llvm/Support/DebugLog.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h.inc"

Classes
class	mlir::impl::GpuEliminateBarriersBase< DerivedT >

Namespaces
namespace	mlir
	Include the generated interface declarations.
namespace	mlir::impl
	Attribute collections provide a dictionary-like interface.

Macros
#define	GEN_PASS_DEF_GPUELIMINATEBARRIERS
#define	DEBUG_TYPE "gpu-erase-barriers"
#define	DEBUG_TYPE_ALIAS "gpu-erase-barries-alias"

Functions
std::unique_ptr<::mlir::Pass >	mlir::impl::createGpuEliminateBarriers ()
std::unique_ptr<::mlir::Pass >	mlir::createGpuEliminateBarriers ()
static bool	isParallelRegionBoundary (Operation *op)
	Returns true if the op is defines the parallel region that is subject to barrier synchronization.
static bool	isSequentialLoopLike (Operation *op)
	Returns true if the op behaves like a sequential loop, e.g., the control flow "wraps around" from the end of the body region back to its start.
static bool	hasSingleExecutionBody (Operation *op)
	Returns true if the regions of the op are guaranteed to be executed at most once.
static bool	producesDistinctBase (Operation *op)
	Returns true if the operation is known to produce a pointer-like object distinct from any other object produced by a similar operation.
static void	addAllValuelessEffects (SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
	Populates effects with all memory effects without associating them to a specific value.
static Value	getBase (Value v)
	Looks through known "view-like" ops to find the base memref.
static bool	isAddressSpacePotentiallyFenced (Attribute memorySpace, std::optional< ArrayRef< gpu::AddressSpaceAttr > > fencedAddressSpaces)
	Returns true if accesses to the given memory space could potentially be fenced by a barrier synchronizing on the given fencedAddressSpaces.
static LogicalResult	effectMightAffectAddressSpaces (const MemoryEffects::EffectInstance &effect, std::optional< ArrayRef< gpu::AddressSpaceAttr > > fencedAddressSpaces)
	Succeeds if the effect operates on a memref whose memory space could be one of the given fenced address spaces.
static bool	isBarrierWithCommonFencedMemory (Operation *op, std::optional< ArrayRef< gpu::AddressSpaceAttr > > fencedAddressSpaces)
	Returns true if op is a BarrierOp that fences any address spaces that could overlap with the given fenced address spaces.
static bool	collectEffects (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, std::optional< ArrayRef< gpu::AddressSpaceAttr > > fencedAddressSpaces, bool ignoreBarriers=true)
	Collect the memory effects of the given op in 'effects'.
static bool	getEffectsBeforeInBlock (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, std::optional< ArrayRef< gpu::AddressSpaceAttr > > fencedAddressSpaces, bool stopAtBarrier)
	Get all effects before the given operation caused by other operations in the same block.
static bool	getEffectsBefore (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, std::optional< ArrayRef< gpu::AddressSpaceAttr > > fencedAddressSpaces, bool stopAtBarrier)
	Collects memory effects from operations that may be executed before op in a trivial structured control flow, e.g., without branches.
static bool	getEffectsAfterInBlock (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, std::optional< ArrayRef< gpu::AddressSpaceAttr > > fencedAddressSpaces, bool stopAtBarrier)
	Get all effects after the given operation caused by other operations in the same block.
static bool	getEffectsAfter (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, std::optional< ArrayRef< gpu::AddressSpaceAttr > > fencedAddressSpaces, bool stopAtBarrier)
	Collects memory effects from operations that may be executed after op in a trivial structured control flow, e.g., without branches.
static bool	isFunctionArgument (Value v)
	Returns true if the value is defined as a function argument.
static Value	propagatesCapture (Operation *op)
	Returns the operand that the operation "propagates" through it for capture purposes.
static std::optional< bool >	getKnownCapturingStatus (Operation *op, Value v)
	Returns true if the given operation is known to capture the given value, false if it is known not to capture the given value, nullopt if neither is known.
static bool	maybeCaptured (Value v)
	Returns true if the value may be captured by any of its users, i.e., if the user may be storing this value into memory.
static bool	mayAlias (Value first, Value second)
	Returns true if two values may be referencing aliasing memory.
static bool	mayAlias (MemoryEffects::EffectInstance a, Value v2)
	Returns true if the effect may be affecting memory aliasing the value.
static bool	mayAlias (MemoryEffects::EffectInstance a, MemoryEffects::EffectInstance b)
	Returns true if the two effects may be affecting aliasing memory.
static bool	haveConflictingEffects (ArrayRef< MemoryEffects::EffectInstance > beforeEffects, ArrayRef< MemoryEffects::EffectInstance > afterEffects)
	Returns true if any of the "before" effect instances has a conflict with any "after" instance for the purpose of barrier elimination.

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "gpu-erase-barriers"

Definition at line 39 of file EliminateBarriers.cpp.

◆ DEBUG_TYPE_ALIAS

#define DEBUG_TYPE_ALIAS "gpu-erase-barries-alias"

Definition at line 40 of file EliminateBarriers.cpp.

Referenced by mayAlias().

◆ GEN_PASS_DEF_GPUELIMINATEBARRIERS

#define GEN_PASS_DEF_GPUELIMINATEBARRIERS

Definition at line 32 of file EliminateBarriers.cpp.

Function Documentation

◆ addAllValuelessEffects()

void addAllValuelessEffects ( SmallVectorImpl< MemoryEffects::EffectInstance > & effects )

static

Populates effects with all memory effects without associating them to a specific value.

Definition at line 74 of file EliminateBarriers.cpp.

References mlir::SideEffects::Effect::get().

Referenced by getEffectsAfter(), and getEffectsBefore().

◆ collectEffects()

bool collectEffects	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		std::optional< ArrayRef< gpu::AddressSpaceAttr > >	fencedAddressSpaces,
		bool	ignoreBarriers = true )

static

Collect the memory effects of the given op in 'effects'.

Returns 'true' if it could extract the effect information from the op, otherwise returns 'false' and conservatively populates the list with all possible effects associated with no particular value or symbol. fencedAddressSpaces is, if given, the set of GPU memory spaces that are being synchronized by the barrier being syrchronized - memory operations where the value being impacted is known and either it or its base value have an address space that is known to be distinct from the ones being synchronized on will not be included in the effect set.

Definition at line 193 of file EliminateBarriers.cpp.

Referenced by mlir::impl::GpuEliminateBarriersBase< DerivedT >::clonePass(), mlir::createGpuEliminateBarriers(), getEffectsAfter(), getEffectsAfterInBlock(), and getEffectsBefore().

◆ effectMightAffectAddressSpaces()

LogicalResult effectMightAffectAddressSpaces	(	const MemoryEffects::EffectInstance &	effect,
		std::optional< ArrayRef< gpu::AddressSpaceAttr > >	fencedAddressSpaces )

static

Succeeds if the effect operates on a memref whose memory space could be one of the given fenced address spaces.

This will both look at the address space of the effect's operand and of the view-like operations that define that memref, so as to inspect any memory-space casts or similar operations (like amdgpu buffer casts) that may provide more information. This assumes that directly-conflicting casts (that is, for example, casting a memref in global memory to make it one in workspace memory) can't happen.

Definition at line 124 of file EliminateBarriers.cpp.

References mlir::Value::getDefiningOp(), mlir::SideEffects::EffectInstance< EffectT >::getValue(), isAddressSpacePotentiallyFenced(), and success().

Referenced by mlir::impl::GpuEliminateBarriersBase< DerivedT >::getName().

◆ getBase()

Value getBase ( Value v )

static

Looks through known "view-like" ops to find the base memref.

Definition at line 83 of file EliminateBarriers.cpp.

References mlir::Value::getDefiningOp().

Referenced by mlir::BlockRange::BlockRange(), llvm::DenseMapInfo< mlir::SuccessorRange >::isEqual(), mlir::MutableOperandRangeRange::join(), mlir::OperandRangeRange::join(), mayAlias(), mlir::OperandRangeRange::OperandRangeRange(), mlir::MutableOperandRangeRange::operator OperandRangeRange(), mlir::TypeRange::TypeRange(), mlir::TypeRange::TypeRange(), and mlir::TypeRange::TypeRange().

◆ getEffectsAfter()

bool getEffectsAfter	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		std::optional< ArrayRef< gpu::AddressSpaceAttr > >	fencedAddressSpaces,
		bool	stopAtBarrier )

static

Collects memory effects from operations that may be executed after op in a trivial structured control flow, e.g., without branches.

Stops at the parallel region boundary or at the barrier operation if stopAtBarrier is set. Returns true if the memory effects added to effects are exact, false if they are a conservative over-approximation. The latter means that effects contain instances not associated with a specific value.

Definition at line 356 of file EliminateBarriers.cpp.

References addAllValuelessEffects(), mlir::WalkResult::advance(), collectEffects(), mlir::Block::front(), mlir::Operation::getBlock(), getEffectsAfter(), getEffectsAfterInBlock(), mlir::Block::getParent(), mlir::Operation::getParentOp(), mlir::Region::hasOneBlock(), hasSingleExecutionBody(), mlir::Operation::hasTrait(), mlir::WalkResult::interrupt(), isBarrierWithCommonFencedMemory(), isParallelRegionBoundary(), isSequentialLoopLike(), and mlir::Operation::walk().

Referenced by getEffectsAfter().

◆ getEffectsAfterInBlock()

bool getEffectsAfterInBlock	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		std::optional< ArrayRef< gpu::AddressSpaceAttr > >	fencedAddressSpaces,
		bool	stopAtBarrier )

static

Get all effects after the given operation caused by other operations in the same block.

That is, this will not consider operations beyond the block.

Definition at line 330 of file EliminateBarriers.cpp.

References mlir::Block::back(), collectEffects(), mlir::Operation::getBlock(), and isBarrierWithCommonFencedMemory().

Referenced by getEffectsAfter().

◆ getEffectsBefore()

bool getEffectsBefore	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		std::optional< ArrayRef< gpu::AddressSpaceAttr > >	fencedAddressSpaces,
		bool	stopAtBarrier )

static

Collects memory effects from operations that may be executed before op in a trivial structured control flow, e.g., without branches.

Stops at the parallel region boundary or at the barrier operation if stopAtBarrier is set. Returns true if the memory effects added to effects are exact, false if they are a conservative over-approximation. The latter means that effects contain instances not associated with a specific value.

Definition at line 264 of file EliminateBarriers.cpp.

References addAllValuelessEffects(), mlir::WalkResult::advance(), collectEffects(), mlir::Operation::getBlock(), getEffectsBefore(), getEffectsBeforeInBlock(), mlir::Block::getParent(), mlir::Operation::getParentOp(), mlir::Block::getTerminator(), mlir::Region::hasOneBlock(), hasSingleExecutionBody(), mlir::Operation::hasTrait(), mlir::WalkResult::interrupt(), isParallelRegionBoundary(), isSequentialLoopLike(), and mlir::Operation::walk().

Referenced by getEffectsBefore().

◆ getEffectsBeforeInBlock()

bool getEffectsBeforeInBlock	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		std::optional< ArrayRef< gpu::AddressSpaceAttr > >	fencedAddressSpaces,
		bool	stopAtBarrier )

static

Get all effects before the given operation caused by other operations in the same block.

That is, this will not consider operations beyond the block.

Definition at line 237 of file EliminateBarriers.cpp.

Referenced by getEffectsBefore().

◆ getKnownCapturingStatus()

std::optional< bool > getKnownCapturingStatus	(	Operation *	op,
		Value	v )

static

Returns true if the given operation is known to capture the given value, false if it is known not to capture the given value, nullopt if neither is known.

Definition at line 449 of file EliminateBarriers.cpp.

Referenced by maybeCaptured().

◆ hasSingleExecutionBody()

bool hasSingleExecutionBody ( Operation * op )

static

Returns true if the regions of the op are guaranteed to be executed at most once.

Thus, if an operation in one of the nested regions of op is executed than so are all the other operations in this region.

Definition at line 61 of file EliminateBarriers.cpp.

Referenced by getEffectsAfter(), and getEffectsBefore().

◆ haveConflictingEffects()

bool haveConflictingEffects	(	ArrayRef< MemoryEffects::EffectInstance >	beforeEffects,
		ArrayRef< MemoryEffects::EffectInstance >	afterEffects )

static

Returns true if any of the "before" effect instances has a conflict with any "after" instance for the purpose of barrier elimination.

The effects are supposed to be limited to a barrier synchronization scope. A conflict exists if effects instances affect aliasing memory locations and at least on of then as a write. As an exception, if the non-write effect is an allocation effect, there is no conflict since we are only expected to see the allocation happening in the same thread and it cannot be accessed from another thread without capture (which we do handle in alias analysis).

Definition at line 611 of file EliminateBarriers.cpp.

References mayAlias().

◆ isAddressSpacePotentiallyFenced()

bool isAddressSpacePotentiallyFenced	(	Attribute	memorySpace,
		std::optional< ArrayRef< gpu::AddressSpaceAttr > >	fencedAddressSpaces )

static

Returns true if accesses to the given memory space could potentially be fenced by a barrier synchronizing on the given fencedAddressSpaces.

If the set of address spaces is not given, it is equal to all possible address spaces. Memory spaces that are not #gpu.address_space are deemed to overlap with all GPU address spaces.

Definition at line 103 of file EliminateBarriers.cpp.

Referenced by effectMightAffectAddressSpaces().

◆ isBarrierWithCommonFencedMemory()

bool isBarrierWithCommonFencedMemory	(	Operation *	op,
		std::optional< ArrayRef< gpu::AddressSpaceAttr > >	fencedAddressSpaces )

static

Returns true if op is a BarrierOp that fences any address spaces that could overlap with the given fenced address spaces.

Definition at line 159 of file EliminateBarriers.cpp.

Referenced by mlir::createGpuEliminateBarriers(), getEffectsAfter(), and getEffectsAfterInBlock().

◆ isFunctionArgument()

bool isFunctionArgument ( Value v )

static

Returns true if the value is defined as a function argument.

Definition at line 429 of file EliminateBarriers.cpp.

Referenced by mayAlias().

◆ isParallelRegionBoundary()

bool isParallelRegionBoundary ( Operation * op )

static

Returns true if the op is defines the parallel region that is subject to barrier synchronization.

Definition at line 47 of file EliminateBarriers.cpp.

References mlir::Operation::hasAttr().

Referenced by getEffectsAfter(), and getEffectsBefore().

◆ isSequentialLoopLike()

bool isSequentialLoopLike ( Operation * op )

static

Returns true if the op behaves like a sequential loop, e.g., the control flow "wraps around" from the end of the body region back to its start.

Definition at line 56 of file EliminateBarriers.cpp.

Referenced by getEffectsAfter(), and getEffectsBefore().

◆ mayAlias() [1/3]

bool mayAlias	(	MemoryEffects::EffectInstance	a,
		MemoryEffects::EffectInstance	b )

static

Returns true if the two effects may be affecting aliasing memory.

If an effect is not associated with any value, it is assumed to affect all memory and therefore aliases with everything. Effects on different resources cannot alias.

Definition at line 589 of file EliminateBarriers.cpp.

References b, mlir::SideEffects::EffectInstance< EffectT >::getResource(), mlir::SideEffects::Resource::getResourceID(), mlir::SideEffects::EffectInstance< EffectT >::getValue(), and mayAlias().

◆ mayAlias() [2/3]

bool mayAlias	(	MemoryEffects::EffectInstance	a,
		Value	v2 )

static

Returns true if the effect may be affecting memory aliasing the value.

If the effect is not associated with any value, it is assumed to affect all memory and therefore aliases with everything.

Definition at line 578 of file EliminateBarriers.cpp.

References mlir::SideEffects::EffectInstance< EffectT >::getValue(), and mayAlias().

◆ mayAlias() [3/3]

bool mayAlias	(	Value	first,
		Value	second )

static

Returns true if two values may be referencing aliasing memory.

This is a rather naive and conservative analysis. Values defined by different allocation-like operations as well as values derived from those by casts and views cannot alias each other. Similarly, values defined by allocations inside a function cannot alias function arguments. Global values cannot alias each other or local allocations. Values that are captured, i.e. themselves potentially stored in memory, are considered as aliasing with everything. This seems sufficient to achieve barrier removal in structured control flow, more complex cases would require a proper dataflow analysis.

Definition at line 510 of file EliminateBarriers.cpp.

References DEBUG_TYPE_ALIAS, getBase(), mlir::Value::getDefiningOp(), isFunctionArgument(), maybeCaptured(), and producesDistinctBase().

Referenced by mlir::impl::SCFParallelLoopFusionBase< DerivedT >::classof(), findUnusedStore(), forwardStoreToLoad(), fuseIfLegal(), mlir::affine::hasNoInterveningEffect(), haveConflictingEffects(), isFusionLegal(), loadCSE(), mayAlias(), mayAlias(), mlir::scf::naivelyFuseParallelOps(), noIncompatibleDataDependencies(), and mlir::scf::parallelForToNestedFors().

◆ maybeCaptured()

bool maybeCaptured ( Value v )

static

Returns true if the value may be captured by any of its users, i.e., if the user may be storing this value into memory.

This makes aliasing analysis more conservative as it cannot assume the pointer-like value is only passed around through SSA use-def.

Definition at line 467 of file EliminateBarriers.cpp.

References mlir::SideEffects::EffectInstance< EffectT >::getEffect(), getKnownCapturingStatus(), mlir::Value::getUsers(), and propagatesCapture().

Referenced by mayAlias().

◆ producesDistinctBase()

bool producesDistinctBase ( Operation * op )

static

Returns true if the operation is known to produce a pointer-like object distinct from any other object produced by a similar operation.

For example, an allocation produces such an object.

Definition at line 68 of file EliminateBarriers.cpp.

Referenced by mayAlias().

◆ propagatesCapture()

Value propagatesCapture ( Operation * op )

static

Returns the operand that the operation "propagates" through it for capture purposes.

That is, if the value produced by this operation is captured, then so is the returned value.

Definition at line 437 of file EliminateBarriers.cpp.

Referenced by maybeCaptured().

Classes

Namespaces

Macros

Functions

Macro Definition Documentation

◆ DEBUG_TYPE

◆ DEBUG_TYPE_ALIAS

◆ GEN_PASS_DEF_GPUELIMINATEBARRIERS

Function Documentation

◆ addAllValuelessEffects()

◆ collectEffects()

◆ effectMightAffectAddressSpaces()

◆ getBase()

◆ getEffectsAfter()

◆ getEffectsAfterInBlock()

◆ getEffectsBefore()

◆ getEffectsBeforeInBlock()

◆ getKnownCapturingStatus()

◆ hasSingleExecutionBody()

◆ haveConflictingEffects()

◆ isAddressSpacePotentiallyFenced()

◆ isBarrierWithCommonFencedMemory()

◆ isFunctionArgument()

◆ isParallelRegionBoundary()

◆ isSequentialLoopLike()

◆ mayAlias() [1/3]

◆ mayAlias() [2/3]

◆ mayAlias() [3/3]

◆ maybeCaptured()

◆ producesDistinctBase()

◆ propagatesCapture()