#include "mlir/Dialect/LLVMIR/Transforms/InlinerInterfaceImpl.h"
#include "mlir/Analysis/SliceWalk.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/IR/Matchers.h"
#include "mlir/Interfaces/DataLayoutInterfaces.h"
#include "mlir/Interfaces/ViewLikeInterface.h"
#include "mlir/Transforms/InliningUtils.h"
#include "llvm/ADT/ScopeExit.h"
#include "llvm/Support/Debug.h"

Macros
#define	DEBUG_TYPE "llvm-inliner"

Functions
static bool	hasLifetimeMarkers (LLVM::AllocaOp allocaOp)
	Check whether the given alloca is an input to a lifetime intrinsic, optionally passing through one or more casts on the way. More...

static void	handleInlinedAllocas (Operation *call, iterator_range< Region::iterator > inlinedBlocks)
	Handles alloca operations in the inlined blocks: More...

static void	deepCloneAliasScopes (iterator_range< Region::iterator > inlinedBlocks)
	Maps all alias scopes in the inlined operations to deep clones of the scopes and domain. More...

static ArrayAttr	concatArrayAttr (ArrayAttr lhs, ArrayAttr rhs)
	Creates a new ArrayAttr by concatenating `lhs` with `rhs`. More...

static FailureOr< SmallVector< Value > >	getUnderlyingObjectSet (Value pointerValue)
	Attempts to return the set of all underlying pointer values that `pointerValue` is based on. More...

static void	createNewAliasScopesFromNoAliasParameter (Operation *call, iterator_range< Region::iterator > inlinedBlocks)
	Creates a new AliasScopeAttr for every noalias parameter and attaches it to the appropriate inlined memory operations in an attempt to preserve the original semantics of the parameter attribute. More...

static void	appendCallOpAliasScopes (Operation *call, iterator_range< Region::iterator > inlinedBlocks)
	Appends any alias scopes of the call operation to any inlined memory operation. More...

static void	handleAliasScopes (Operation *call, iterator_range< Region::iterator > inlinedBlocks)
	Handles all interactions with alias scopes during inlining. More...

static void	handleAccessGroups (Operation *call, iterator_range< Region::iterator > inlinedBlocks)
	Appends any access groups of the call operation to any inlined memory operation. More...

static void	handleLoopAnnotations (Operation *call, iterator_range< Region::iterator > inlinedBlocks)
	Updates locations inside loop annotations to reflect that they were inlined. More...

static uint64_t	tryToEnforceAllocaAlignment (LLVM::AllocaOp alloca, uint64_t requestedAlignment, DataLayout const &dataLayout)
	If `requestedAlignment` is higher than the alignment specified on `alloca`, realigns `alloca` if this does not exceed the natural stack alignment. More...

static uint64_t	tryToEnforceAlignment (Value value, uint64_t requestedAlignment, DataLayout const &dataLayout)
	Tries to find and return the alignment of the pointer `value` by looking for an alignment attribute on the defining allocation op or function argument. More...

static Value	handleByValArgumentInit (OpBuilder &builder, Location loc, Value argument, Type elementType, uint64_t elementTypeSize, uint64_t targetAlignment)
	Introduces a new alloca and copies the memory pointed to by `argument` to the address of the new alloca, then returns the value of the new alloca. More...

static Value	handleByValArgument (OpBuilder &builder, Operation *callable, Value argument, Type elementType, uint64_t requestedAlignment)
	Handles a function argument marked with the byval attribute by introducing a memcpy or realigning the defining operation, if required either due to the pointee being writeable in the callee, and/or due to an alignment mismatch. More...

Macro Definition Documentation

◆ DEBUG_TYPE

#define DEBUG_TYPE "llvm-inliner"

Definition at line 24 of file InlinerInterfaceImpl.cpp.

Function Documentation

◆ appendCallOpAliasScopes()

static void appendCallOpAliasScopes	(	Operation *	call,
		iterator_range< Region::iterator >	inlinedBlocks
	)

static

Appends any alias scopes of the call operation to any inlined memory operation.

Definition at line 423 of file InlinerInterfaceImpl.cpp.

References concatArrayAttr().

Referenced by handleAliasScopes().

◆ concatArrayAttr()

static ArrayAttr concatArrayAttr	(	ArrayAttr	lhs,
		ArrayAttr	rhs
	)

static

Creates a new ArrayAttr by concatenating lhs with rhs.

Returns null if both parameters are null. If only one attribute is null, return the other.

Definition at line 214 of file InlinerInterfaceImpl.cpp.

References mlir::get().

Referenced by appendCallOpAliasScopes(), createNewAliasScopesFromNoAliasParameter(), and handleAccessGroups().

◆ createNewAliasScopesFromNoAliasParameter()

static void createNewAliasScopesFromNoAliasParameter	(	Operation *	call,
		iterator_range< Region::iterator >	inlinedBlocks
	)

static

Creates a new AliasScopeAttr for every noalias parameter and attaches it to the appropriate inlined memory operations in an attempt to preserve the original semantics of the parameter attribute.

Definition at line 266 of file InlinerInterfaceImpl.cpp.

References concatArrayAttr(), copy(), mlir::OpBuilder::create(), mlir::get(), mlir::Operation::getContext(), mlir::Operation::getLoc(), getUnderlyingObjectSet(), mlir::Operation::getUsers(), mlir::m_Constant(), mlir::matchPattern(), and mlir::Operation::walk().

Referenced by handleAliasScopes().

◆ deepCloneAliasScopes()

static void deepCloneAliasScopes ( iterator_range< Region::iterator > inlinedBlocks )

static

Maps all alias scopes in the inlined operations to deep clones of the scopes and domain.

This is required for code such as foo(a, b); foo(a2, b2); to not incorrectly return noalias for e.g. operations on a and a2.

Definition at line 155 of file InlinerInterfaceImpl.cpp.

References mlir::AttrTypeWalker::addWalk(), mlir::get(), and mlir::AttrTypeWalker::walk().

Referenced by handleAliasScopes().

◆ getUnderlyingObjectSet()

static FailureOr<SmallVector<Value> > getUnderlyingObjectSet ( Value pointerValue )

static

Attempts to return the set of all underlying pointer values that pointerValue is based on.

This function traverses through select operations and block arguments.

Definition at line 230 of file InlinerInterfaceImpl.cpp.

References mlir::WalkContinuation::advanceTo(), mlir::getControlFlowPredecessors(), mlir::Value::getDefiningOp(), mlir::WalkContinuation::interrupt(), mlir::WalkContinuation::skip(), mlir::walkSlice(), and mlir::WalkContinuation::wasInterrupted().

Referenced by createNewAliasScopesFromNoAliasParameter().

◆ handleAccessGroups()

static void handleAccessGroups	(	Operation *	call,
		iterator_range< Region::iterator >	inlinedBlocks
	)

static

Appends any access groups of the call operation to any inlined memory operation.

Definition at line 461 of file InlinerInterfaceImpl.cpp.

References concatArrayAttr().

◆ handleAliasScopes()

static void handleAliasScopes	(	Operation *	call,
		iterator_range< Region::iterator >	inlinedBlocks
	)

static

Handles all interactions with alias scopes during inlining.

Definition at line 452 of file InlinerInterfaceImpl.cpp.

References appendCallOpAliasScopes(), createNewAliasScopesFromNoAliasParameter(), and deepCloneAliasScopes().

◆ handleByValArgument()

static Value handleByValArgument	(	OpBuilder &	builder,
		Operation *	callable,
		Value	argument,
		Type	elementType,
		uint64_t	requestedAlignment
	)

static

Handles a function argument marked with the byval attribute by introducing a memcpy or realigning the defining operation, if required either due to the pointee being writeable in the callee, and/or due to an alignment mismatch.

requestedAlignment specifies the alignment set in the "align" argument attribute (or 1 if no align attribute was set).

Definition at line 623 of file InlinerInterfaceImpl.cpp.

References mlir::DataLayout::closest(), mlir::DataLayout::getTypeABIAlignment(), mlir::DataLayout::getTypeSize(), handleByValArgumentInit(), max(), and tryToEnforceAlignment().

◆ handleByValArgumentInit()

static Value handleByValArgumentInit	(	OpBuilder &	builder,
		Location	loc,
		Value	argument,
		Type	elementType,
		uint64_t	elementTypeSize,
		uint64_t	targetAlignment
	)

static

Introduces a new alloca and copies the memory pointed to by argument to the address of the new alloca, then returns the value of the new alloca.

Definition at line 593 of file InlinerInterfaceImpl.cpp.

References mlir::Region::begin(), mlir::OpBuilder::create(), mlir::Builder::getI64IntegerAttr(), mlir::Builder::getI64Type(), mlir::Value::getParentRegion(), mlir::Value::getType(), and mlir::OpBuilder::setInsertionPointToStart().

Referenced by handleByValArgument().

◆ handleInlinedAllocas()

static void handleInlinedAllocas	(	Operation *	call,
		iterator_range< Region::iterator >	inlinedBlocks
	)

static

Handles alloca operations in the inlined blocks:

Moves all alloca operations with a constant size in the former entry block of the callee into the entry block of the caller, so they become part of the function prologue/epilogue during code generation.
Inserts lifetime intrinsics that limit the scope of inlined static allocas to the inlined blocks.
Inserts StackSave and StackRestore operations if dynamic allocas were inlined.

Definition at line 53 of file InlinerInterfaceImpl.cpp.

References mlir::Block::begin(), mlir::OpBuilder::create(), mlir::Region::front(), mlir::get(), mlir::Operation::getContext(), mlir::Operation::getLoc(), mlir::Block::getOps(), mlir::Operation::getParentOp(), mlir::Operation::getParentRegion(), hasLifetimeMarkers(), mlir::m_Constant(), mlir::matchPattern(), mlir::Operation::moveAfter(), mlir::OpBuilder::setInsertionPoint(), and mlir::OpBuilder::setInsertionPointToStart().

◆ handleLoopAnnotations()

static void handleLoopAnnotations	(	Operation *	call,
		iterator_range< Region::iterator >	inlinedBlocks
	)

static

Updates locations inside loop annotations to reflect that they were inlined.

Definition at line 482 of file InlinerInterfaceImpl.cpp.

References mlir::detail::AttrTypeReplacerBase< Concrete >::addReplacement(), mlir::WalkResult::advance(), mlir::get(), mlir::Operation::getLoc(), mlir::Operation::getParentOfType(), and mlir::detail::AttrTypeReplacerBase< Concrete >::recursivelyReplaceElementsIn().

◆ hasLifetimeMarkers()

static bool hasLifetimeMarkers ( LLVM::AllocaOp allocaOp )

static

Check whether the given alloca is an input to a lifetime intrinsic, optionally passing through one or more casts on the way.

This is not transitive through block arguments.

Definition at line 31 of file InlinerInterfaceImpl.cpp.

References mlir::Operation::getUsers().

Referenced by handleInlinedAllocas().

◆ tryToEnforceAlignment()

static uint64_t tryToEnforceAlignment	(	Value	value,
		uint64_t	requestedAlignment,
		DataLayout const &	dataLayout
	)

static

Tries to find and return the alignment of the pointer value by looking for an alignment attribute on the defining allocation op or function argument.

If the found alignment is lower than requestedAlignment, tries to realign the pointer, then returns the resulting post-alignment, regardless of whether it was realigned or not. If no existing alignment attribute is found, returns 1 (i.e., assume that no alignment is guaranteed).

Definition at line 563 of file InlinerInterfaceImpl.cpp.

References mlir::Value::getDefiningOp(), mlir::Value::getParentBlock(), mlir::Block::getParentOp(), and tryToEnforceAllocaAlignment().

Referenced by handleByValArgument().

◆ tryToEnforceAllocaAlignment()

static uint64_t tryToEnforceAllocaAlignment	(	LLVM::AllocaOp	alloca,
		uint64_t	requestedAlignment,
		DataLayout const &	dataLayout
	)

static

If requestedAlignment is higher than the alignment specified on alloca, realigns alloca if this does not exceed the natural stack alignment.

Returns the post-alignment of alloca, whether it was realigned or not.

Definition at line 534 of file InlinerInterfaceImpl.cpp.

References mlir::DataLayout::getStackAlignment().

Referenced by tryToEnforceAlignment().

Macros

Functions

Macro Definition Documentation

◆ DEBUG_TYPE

Function Documentation

◆ appendCallOpAliasScopes()

◆ concatArrayAttr()

◆ createNewAliasScopesFromNoAliasParameter()

◆ deepCloneAliasScopes()

◆ getUnderlyingObjectSet()

◆ handleAccessGroups()

◆ handleAliasScopes()

◆ handleByValArgument()

◆ handleByValArgumentInit()

◆ handleInlinedAllocas()

◆ handleLoopAnnotations()

◆ hasLifetimeMarkers()

◆ tryToEnforceAlignment()

◆ tryToEnforceAllocaAlignment()