#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/GPU/IR/GPUDialect.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/IR/Operation.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Transforms/GreedyPatternRewriteDriver.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/Debug.h"
#include "mlir/Dialect/GPU/Transforms/Passes.h.inc"

Namespaces
	mlir
	Include the generated interface declarations.

Macros
#define	GEN_PASS_DEF_GPUELIMINATEBARRIERS

#define	DEBUG_TYPE "gpu-erase-barriers"

#define	DEBUG_TYPE_ALIAS "gpu-erase-barries-alias"

#define	DBGS() (llvm::dbgs() << '[' << DEBUG_TYPE << "] ")

#define	DBGS_ALIAS() (llvm::dbgs() << '[' << DEBUG_TYPE_ALIAS << "] ")

Functions
static bool	isKnownNoEffectsOpWithoutInterface (Operation *op)
	Implement the MemoryEffectsOpInterface in the suitable way. More...

static bool	isParallelRegionBoundary (Operation *op)
	Returns `true` if the op is defines the parallel region that is subject to barrier synchronization. More...

static bool	isSequentialLoopLike (Operation *op)
	Returns `true` if the op behaves like a sequential loop, e.g., the control flow "wraps around" from the end of the body region back to its start. More...

static bool	hasSingleExecutionBody (Operation *op)
	Returns `true` if the regions of the op are guaranteed to be executed at most once. More...

static bool	producesDistinctBase (Operation *op)
	Returns `true` if the operation is known to produce a pointer-like object distinct from any other object produced by a similar operation. More...

static void	addAllValuelessEffects (SmallVectorImpl< MemoryEffects::EffectInstance > &effects)
	Populates `effects` with all memory effects without associating them to a specific value. More...

static bool	collectEffects (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool ignoreBarriers=true)
	Collect the memory effects of the given op in 'effects'. More...

static bool	getEffectsBeforeInBlock (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool stopAtBarrier)
	Get all effects before the given operation caused by other operations in the same block. More...

static bool	getEffectsBefore (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool stopAtBarrier)
	Collects memory effects from operations that may be executed before `op` in a trivial structured control flow, e.g., without branches. More...

static bool	getEffectsAfterInBlock (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool stopAtBarrier)
	Get all effects after the given operation caused by other operations in the same block. More...

static bool	getEffectsAfter (Operation *op, SmallVectorImpl< MemoryEffects::EffectInstance > &effects, bool stopAtBarrier)
	Collects memory effects from operations that may be executed after `op` in a trivial structured control flow, e.g., without branches. More...

static Value	getBase (Value v)
	Looks through known "view-like" ops to find the base memref. More...

static bool	isFunctionArgument (Value v)
	Returns `true` if the value is defined as a function argument. More...

static Value	propagatesCapture (Operation *op)
	Returns the operand that the operation "propagates" through it for capture purposes. More...

static std::optional< bool >	getKnownCapturingStatus (Operation *op, Value v)
	Returns `true` if the given operation is known to capture the given value, `false` if it is known not to capture the given value, `nullopt` if neither is known. More...

static bool	maybeCaptured (Value v)
	Returns `true` if the value may be captured by any of its users, i.e., if the user may be storing this value into memory. More...

static bool	mayAlias (Value first, Value second)
	Returns true if two values may be referencing aliasing memory. More...

static bool	mayAlias (MemoryEffects::EffectInstance a, Value v2)
	Returns `true` if the effect may be affecting memory aliasing the value. More...

static bool	mayAlias (MemoryEffects::EffectInstance a, MemoryEffects::EffectInstance b)
	Returns `true` if the two effects may be affecting aliasing memory. More...

static bool	haveConflictingEffects (ArrayRef< MemoryEffects::EffectInstance > beforeEffects, ArrayRef< MemoryEffects::EffectInstance > afterEffects)
	Returns `true` if any of the "before" effect instances has a conflict with any "after" instance for the purpose of barrier elimination. More...

Macro Definition Documentation

◆ DBGS

#define DBGS ( ) (llvm::dbgs() << '[' << DEBUG_TYPE << "] ")

Definition at line 41 of file EliminateBarriers.cpp.

◆ DBGS_ALIAS

#define DBGS_ALIAS ( ) (llvm::dbgs() << '[' << DEBUG_TYPE_ALIAS << "] ")

Definition at line 42 of file EliminateBarriers.cpp.

◆ DEBUG_TYPE

#define DEBUG_TYPE "gpu-erase-barriers"

Definition at line 38 of file EliminateBarriers.cpp.

◆ DEBUG_TYPE_ALIAS

#define DEBUG_TYPE_ALIAS "gpu-erase-barries-alias"

Definition at line 39 of file EliminateBarriers.cpp.

◆ GEN_PASS_DEF_GPUELIMINATEBARRIERS

#define GEN_PASS_DEF_GPUELIMINATEBARRIERS

Definition at line 31 of file EliminateBarriers.cpp.

Function Documentation

◆ addAllValuelessEffects()

static void addAllValuelessEffects ( SmallVectorImpl< MemoryEffects::EffectInstance > & effects )

static

Populates effects with all memory effects without associating them to a specific value.

Definition at line 83 of file EliminateBarriers.cpp.

Referenced by collectEffects(), getEffectsAfter(), and getEffectsBefore().

◆ collectEffects()

static bool collectEffects	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		bool	ignoreBarriers = `true`
	)

static

Collect the memory effects of the given op in 'effects'.

Returns 'true' if it could extract the effect information from the op, otherwise returns 'false' and conservatively populates the list with all possible effects associated with no particular value or symbol.

Definition at line 96 of file EliminateBarriers.cpp.

References addAllValuelessEffects(), mlir::Operation::getRegions(), mlir::Operation::hasTrait(), and isKnownNoEffectsOpWithoutInterface().

Referenced by getEffectsAfter(), getEffectsAfterInBlock(), getEffectsBefore(), and getEffectsBeforeInBlock().

◆ getBase()

static Value getBase ( Value v )

static

Looks through known "view-like" ops to find the base memref.

Definition at line 321 of file EliminateBarriers.cpp.

References mlir::Value::getDefiningOp().

Referenced by llvm::DenseMapInfo< mlir::SuccessorRange >::isEqual(), mlir::OperandRangeRange::join(), mlir::MutableOperandRangeRange::join(), mayAlias(), and mlir::MutableOperandRangeRange::operator OperandRangeRange().

◆ getEffectsAfter()

static bool getEffectsAfter	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		bool	stopAtBarrier
	)

static

Collects memory effects from operations that may be executed after op in a trivial structured control flow, e.g., without branches.

Stops at the parallel region boundary or at the barrier operation if stopAtBarrier is set. Returns true if the memory effects added to effects are exact, false if they are a conservative over-approximation. The latter means that effects contain instances not associated with a specific value.

Definition at line 255 of file EliminateBarriers.cpp.

References addAllValuelessEffects(), mlir::WalkResult::advance(), collectEffects(), mlir::Block::front(), mlir::Operation::getBlock(), mlir::Region::getBlocks(), getEffectsAfterInBlock(), mlir::Block::getParent(), mlir::Operation::getParentOp(), hasSingleExecutionBody(), mlir::WalkResult::interrupt(), isParallelRegionBoundary(), isSequentialLoopLike(), and mlir::Operation::walk().

◆ getEffectsAfterInBlock()

static bool getEffectsAfterInBlock	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		bool	stopAtBarrier
	)

static

Get all effects after the given operation caused by other operations in the same block.

That is, this will not consider operations beyond the block.

Definition at line 229 of file EliminateBarriers.cpp.

References mlir::Block::back(), collectEffects(), and mlir::Operation::getBlock().

Referenced by getEffectsAfter().

◆ getEffectsBefore()

static bool getEffectsBefore	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		bool	stopAtBarrier
	)

static

Collects memory effects from operations that may be executed before op in a trivial structured control flow, e.g., without branches.

Stops at the parallel region boundary or at the barrier operation if stopAtBarrier is set. Returns true if the memory effects added to effects are exact, false if they are a conservative over-approximation. The latter means that effects contain instances not associated with a specific value.

Definition at line 165 of file EliminateBarriers.cpp.

References addAllValuelessEffects(), mlir::WalkResult::advance(), collectEffects(), mlir::Operation::getBlock(), mlir::Region::getBlocks(), getEffectsBeforeInBlock(), mlir::Block::getParent(), mlir::Operation::getParentOp(), mlir::Block::getTerminator(), hasSingleExecutionBody(), mlir::WalkResult::interrupt(), isParallelRegionBoundary(), isSequentialLoopLike(), and mlir::Operation::walk().

◆ getEffectsBeforeInBlock()

static bool getEffectsBeforeInBlock	(	Operation *	op,
		SmallVectorImpl< MemoryEffects::EffectInstance > &	effects,
		bool	stopAtBarrier
	)

static

Get all effects before the given operation caused by other operations in the same block.

That is, this will not consider operations beyond the block.

Definition at line 138 of file EliminateBarriers.cpp.

References collectEffects(), mlir::Block::front(), and mlir::Operation::getBlock().

Referenced by getEffectsBefore().

◆ getKnownCapturingStatus()

static std::optional<bool> getKnownCapturingStatus	(	Operation *	op,
		Value	v
	)

static

Returns true if the given operation is known to capture the given value, false if it is known not to capture the given value, nullopt if neither is known.

Definition at line 372 of file EliminateBarriers.cpp.

Referenced by maybeCaptured().

◆ hasSingleExecutionBody()

static bool hasSingleExecutionBody ( Operation * op )

static

Returns true if the regions of the op are guaranteed to be executed at most once.

Thus, if an operation in one of the nested regions of op is executed than so are all the other operations in this region.

Definition at line 70 of file EliminateBarriers.cpp.

Referenced by getEffectsAfter(), and getEffectsBefore().

◆ haveConflictingEffects()

static bool haveConflictingEffects	(	ArrayRef< MemoryEffects::EffectInstance >	beforeEffects,
		ArrayRef< MemoryEffects::EffectInstance >	afterEffects
	)

static

Returns true if any of the "before" effect instances has a conflict with any "after" instance for the purpose of barrier elimination.

The effects are supposed to be limited to a barrier synchronization scope. A conflict exists if effects instances affect aliasing memory locations and at least on of then as a write. As an exception, if the non-write effect is an allocation effect, there is no conflict since we are only expected to see the allocation happening in the same thread and it cannot be accessed from another thread without capture (which we do handle in alias analysis).

Definition at line 542 of file EliminateBarriers.cpp.

References DBGS, and mayAlias().

◆ isFunctionArgument()

static bool isFunctionArgument ( Value v )

static

Returns true if the value is defined as a function argument.

Definition at line 350 of file EliminateBarriers.cpp.

Referenced by mayAlias().

◆ isKnownNoEffectsOpWithoutInterface()

static bool isKnownNoEffectsOpWithoutInterface ( Operation * op )

static

Implement the MemoryEffectsOpInterface in the suitable way.

Definition at line 48 of file EliminateBarriers.cpp.

Referenced by collectEffects().

◆ isParallelRegionBoundary()

static bool isParallelRegionBoundary ( Operation * op )

static

Returns true if the op is defines the parallel region that is subject to barrier synchronization.

Definition at line 56 of file EliminateBarriers.cpp.

References mlir::Operation::hasAttr().

Referenced by getEffectsAfter(), and getEffectsBefore().

◆ isSequentialLoopLike()

static bool isSequentialLoopLike ( Operation * op )

static

Returns true if the op behaves like a sequential loop, e.g., the control flow "wraps around" from the end of the body region back to its start.

Definition at line 65 of file EliminateBarriers.cpp.

Referenced by getEffectsAfter(), and getEffectsBefore().

◆ mayAlias() [1/3]

static bool mayAlias	(	MemoryEffects::EffectInstance	a,
		MemoryEffects::EffectInstance	b
	)

static

Returns true if the two effects may be affecting aliasing memory.

If an effect is not associated with any value, it is assumed to affect all memory and therefore aliases with everything. Effects on different resources cannot alias.

Definition at line 521 of file EliminateBarriers.cpp.

References mlir::SideEffects::EffectInstance< EffectT >::getResource(), mlir::SideEffects::Resource::getResourceID(), mlir::SideEffects::EffectInstance< EffectT >::getValue(), and mayAlias().

◆ mayAlias() [2/3]

static bool mayAlias	(	MemoryEffects::EffectInstance	a,
		Value	v2
	)

static

Returns true if the effect may be affecting memory aliasing the value.

If the effect is not associated with any value, it is assumed to affect all memory and therefore aliases with everything.

Definition at line 510 of file EliminateBarriers.cpp.

References mlir::SideEffects::EffectInstance< EffectT >::getValue(), and mayAlias().

◆ mayAlias() [3/3]

static bool mayAlias	(	Value	first,
		Value	second
	)

static

Returns true if two values may be referencing aliasing memory.

This is a rather naive and conservative analysis. Values defined by different allocation-like operations as well as values derived from those by casts and views cannot alias each other. Similarly, values defined by allocations inside a function cannot alias function arguments. Global values cannot alias each other or local allocations. Values that are captured, i.e. themselves potentially stored in memory, are considered as aliasing with everything. This seems sufficient to achieve barrier removal in structured control flow, more complex cases would require a proper dataflow analysis.

Definition at line 433 of file EliminateBarriers.cpp.

References DBGS_ALIAS, DEBUG_TYPE_ALIAS, getBase(), mlir::Value::getDefiningOp(), isFunctionArgument(), maybeCaptured(), and producesDistinctBase().

Referenced by mlir::affine::affineScalarReplace(), findUnusedStore(), forwardStoreToLoad(), fuseIfLegal(), mlir::affine::hasNoInterveningEffect(), haveConflictingEffects(), haveNoReadsAfterWriteExceptSameIndex(), isFusionLegal(), loadCSE(), mayAlias(), mlir::scf::naivelyFuseParallelOps(), and verifyDependencies().

◆ maybeCaptured()

static bool maybeCaptured ( Value v )

static

Returns true if the value may be captured by any of its users, i.e., if the user may be storing this value into memory.

This makes aliasing analysis more conservative as it cannot assume the pointer-like value is only passed around through SSA use-def.

Definition at line 390 of file EliminateBarriers.cpp.

References mlir::SideEffects::EffectInstance< EffectT >::getEffect(), getKnownCapturingStatus(), mlir::Value::getUsers(), and propagatesCapture().

Referenced by mayAlias().

◆ producesDistinctBase()

static bool producesDistinctBase ( Operation * op )

static

Returns true if the operation is known to produce a pointer-like object distinct from any other object produced by a similar operation.

For example, an allocation produces such an object.

Definition at line 77 of file EliminateBarriers.cpp.

Referenced by mayAlias().

◆ propagatesCapture()

static Value propagatesCapture ( Operation * op )

static

Returns the operand that the operation "propagates" through it for capture purposes.

That is, if the value produced by this operation is captured, then so is the returned value.

Definition at line 358 of file EliminateBarriers.cpp.

References mlir::xegpu::transpose().

Referenced by maybeCaptured().

Namespaces

Macros

Functions

Macro Definition Documentation

◆ DBGS

◆ DBGS_ALIAS

◆ DEBUG_TYPE

◆ DEBUG_TYPE_ALIAS

◆ GEN_PASS_DEF_GPUELIMINATEBARRIERS

Function Documentation

◆ addAllValuelessEffects()

◆ collectEffects()

◆ getBase()

◆ getEffectsAfter()

◆ getEffectsAfterInBlock()

◆ getEffectsBefore()

◆ getEffectsBeforeInBlock()

◆ getKnownCapturingStatus()

◆ hasSingleExecutionBody()

◆ haveConflictingEffects()

◆ isFunctionArgument()

◆ isKnownNoEffectsOpWithoutInterface()

◆ isParallelRegionBoundary()

◆ isSequentialLoopLike()

◆ mayAlias() [1/3]

◆ mayAlias() [2/3]

◆ mayAlias() [3/3]

◆ maybeCaptured()

◆ producesDistinctBase()

◆ propagatesCapture()