#include "mlir/Dialect/Linalg/IR/Linalg.h"
#include "mlir/AsmParser/AsmParser.h"
#include "mlir/Dialect/Affine/IR/AffineOps.h"
#include "mlir/Dialect/Arith/IR/Arith.h"
#include "mlir/Dialect/Arith/Utils/Utils.h"
#include "mlir/Dialect/Complex/IR/Complex.h"
#include "mlir/Dialect/Math/IR/Math.h"
#include "mlir/Dialect/MemRef/IR/MemRef.h"
#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/SparseTensor/IR/SparseTensor.h"
#include "mlir/Dialect/Tensor/IR/Tensor.h"
#include "mlir/Dialect/Utils/IndexingUtils.h"
#include "mlir/Dialect/Utils/ReshapeOpsUtils.h"
#include "mlir/Dialect/Utils/StaticValueUtils.h"
#include "mlir/IR/AffineExprVisitor.h"
#include "mlir/IR/AffineMap.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/BuiltinTypeInterfaces.h"
#include "mlir/IR/Matchers.h"
#include "mlir/IR/OpImplementation.h"
#include "mlir/IR/OperationSupport.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Interfaces/InferTypeOpInterface.h"
#include "llvm/ADT/DenseMap.h"
#include "llvm/ADT/SmallSet.h"
#include "llvm/ADT/StringSet.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/Support/FormatVariadic.h"
#include "llvm/Support/MathExtras.h"
#include "llvm/Support/raw_ostream.h"
#include <optional>
#include "mlir/Dialect/Linalg/IR/LinalgNamedStructuredOps.yamlgen.cpp.inc"
#include "mlir/Dialect/Linalg/IR/LinalgOps.cpp.inc"
#include "mlir/Dialect/Linalg/IR/LinalgStructuredOps.cpp.inc"

Macros
#define	GET_OP_CLASSES

#define	GET_OP_CLASSES

Typedefs
using	RegionBuilderFn = llvm::function_ref< void(ImplicitLocOpBuilder &, Block &, ArrayRef< NamedAttribute >)>

Functions
static OpFoldResult	getDimValue (OpBuilder &builder, Location loc, Value v, int64_t dim)
	Return a `memref.dim` or `tensor.dim` for the shape of `v` at `dim`. More...

static Value	getSlice (OpBuilder &b, Location loc, Value source, ArrayRef< OpFoldResult > offsets, ArrayRef< OpFoldResult > sizes, ArrayRef< OpFoldResult > strides)
	Returns a memref.subview or a tensor.extract_slice based on the type of the `source`. More...

static void	fillStructuredOpRegion (OpBuilder &opBuilder, Region &region, TypeRange inputTypes, TypeRange outputTypes, ArrayRef< NamedAttribute > attrs, RegionBuilderFn regionBuilder)
	Fills the region of a structured operation using the provided `regionBuilder`. More...

static void	buildStructuredOp (OpBuilder &b, OperationState &state, std::optional< TypeRange > resultTensorTypes, ValueRange inputs, ValueRange outputs, ArrayRef< NamedAttribute > attributes, RegionBuilderFn regionBuilder)
	Creates a structured operation given `inputs`, `outputs`, and `attributes`. More...

static ParseResult	parseCommonStructuredOpParts (OpAsmParser &parser, OperationState &result, SmallVectorImpl< Type > &inputTypes, SmallVectorImpl< Type > &outputTypes, bool addOperandSegmentSizes=true)
	Common parsing used for both named structured ops created by ods-gen and by manually defined C++ ops. More...

static void	printCommonStructuredOpParts (OpAsmPrinter &p, ValueRange inputs, ValueRange outputs)

static ParseResult	parseNamedStructuredOpRegion (OpAsmParser &parser, Region &region, unsigned numRegionArgs, TypeRange inputTypes, TypeRange outputTypes, ArrayRef< NamedAttribute > attrs, RegionBuilderFn regionBuilder)

static ParseResult	parseNamedStructuredOpResults (OpAsmParser &parser, SmallVectorImpl< Type > &resultTypes)

static ParseResult	parseNamedStructuredOp (OpAsmParser &parser, OperationState &result, unsigned numRegionArgs, RegionBuilderFn regionBuilder)

static void	printNamedStructuredOpResults (OpAsmPrinter &p, TypeRange resultTypes)

static void	printNamedStructuredOp (OpAsmPrinter &p, Operation *op, ValueRange inputs, ValueRange outputs)

static void	buildGenericRegion (OpBuilder &builder, Location loc, Region &region, ValueRange inputs, ValueRange outputs, function_ref< void(OpBuilder &, Location, ValueRange)> bodyBuild)

static void	getGenericEffectsImpl (SmallVectorImpl< SideEffects::EffectInstance< MemoryEffects::Effect >> &effects, ValueRange results, const ValueRange inputOperands, ValueRange outputOperands)

static ParseResult	parseDstStyleOp (OpAsmParser &parser, OperationState &result, function_ref< ParseResult(OpAsmParser &, NamedAttrList &)> parseAttrsFn=nullptr)

static void	addBodyWithPayloadOp (OpAsmParser &parser, OperationState &result, const OperationName &payloadOpName, const NamedAttrList &payloadOpAttrs, ArrayRef< Value > operands, bool initFirst=false)

static Operation *	findPayloadOp (Block *body, bool initFirst=false)

void	printShortForm (OpAsmPrinter &p, Operation *payloadOp)

static ParseResult	parseDenseI64ArrayAttr (OpAsmParser &parser, NamedAttrList &attributes, StringRef attributeName)

static void	printDenseI64ArrayAttr (OpAsmPrinter &p, StringRef attributeName, ArrayRef< int64_t > attributeValue)

static void	buildIdentityRegion (OpBuilder &builder, Location loc, Region &region, ValueRange inputs, ValueRange outputs)

static LogicalResult	verifyYield (linalg::YieldOp op, LinalgOp linalgOp)

static LogicalResult	appendMangledType (llvm::raw_string_ostream &ss, Type t)

static std::tuple< SmallVector< utils::IteratorType >, SmallVector< AffineMap > >	computeIteratorTypesAndIndexingMaps (OpBuilder &builder, int64_t inputRank, int64_t dim, bool allParallel=false)

template<typename T >
static Value	reduce (OpBuilder &builder, Location loc, Value input, Value output, int64_t dim)

static Value	buildSubAndExpOp (OpBuilder &builder, Location loc, Value input, Value max, Value output, int64_t dim)
	Produce a linalg generic that computes the second step of the softmax decomposition: res = exp(input - max), where `max` is the max of `input` on dimension `dim`. More...

static Value	buildDivOp (OpBuilder &builder, Location loc, Value numerator, Value denominator, Value output, int64_t dim)
	Produce a linalg generic that computes the final step of the softmax decomposition. More...

Macro Definition Documentation

◆ GET_OP_CLASSES [1/2]

#define GET_OP_CLASSES

Definition at line 2028 of file LinalgOps.cpp.

◆ GET_OP_CLASSES [2/2]

#define GET_OP_CLASSES

Definition at line 2028 of file LinalgOps.cpp.

Typedef Documentation

◆ RegionBuilderFn

using RegionBuilderFn = llvm::function_ref<void(ImplicitLocOpBuilder &, Block &, ArrayRef<NamedAttribute>)>

Definition at line 110 of file LinalgOps.cpp.

Function Documentation

◆ addBodyWithPayloadOp()

static void addBodyWithPayloadOp	(	OpAsmParser &	parser,
		OperationState &	result,
		const OperationName &	payloadOpName,
		const NamedAttrList &	payloadOpAttrs,
		ArrayRef< Value >	operands,
		bool	initFirst = `false`
	)

static

Definition at line 1253 of file LinalgOps.cpp.

References mlir::Block::addArgument(), mlir::OperationState::addRegion(), mlir::OpBuilder::create(), mlir::Region::emplaceBlock(), mlir::Block::getArguments(), mlir::AsmParser::getContext(), mlir::Operation::getResults(), mlir::Builder::getStringAttr(), mlir::OperationName::getStringRef(), mlir::Value::getType(), mlir::Builder::getUnknownLoc(), mlir::OperationState::location, and mlir::OpBuilder::setInsertionPointToStart().

◆ appendMangledType()

static LogicalResult appendMangledType	(	llvm::raw_string_ostream &	ss,
		Type	t
	)

static

Definition at line 2059 of file LinalgOps.cpp.

References mlir::failed(), mlir::failure(), mlir::Type::isSignlessIntOrIndexOrFloat(), and mlir::success().

◆ buildDivOp()

static Value buildDivOp	(	OpBuilder &	builder,
		Location	loc,
		Value	numerator,
		Value	denominator,
		Value	output,
		int64_t	dim
	)

static

Produce a linalg generic that computes the final step of the softmax decomposition.

Returns: linalg.generic ins(numerator, denominator) outs(output) { yield n / d }

Definition at line 2564 of file LinalgOps.cpp.

References computeIteratorTypesAndIndexingMaps(), mlir::OpBuilder::create(), mlir::Operation::getResult(), and mlir::Value::getType().

◆ buildGenericRegion()

static void buildGenericRegion	(	OpBuilder &	builder,
		Location	loc,
		Region &	region,
		ValueRange	inputs,
		ValueRange	outputs,
		function_ref< void(OpBuilder &, Location, ValueRange)>	bodyBuild
	)

static

Definition at line 849 of file LinalgOps.cpp.

References mlir::OpBuilder::createBlock(), mlir::Region::end(), mlir::Block::getArguments(), and mlir::getElementTypeOrSelf().

Referenced by buildIdentityRegion().

◆ buildIdentityRegion()

static void buildIdentityRegion	(	OpBuilder &	builder,
		Location	loc,
		Region &	region,
		ValueRange	inputs,
		ValueRange	outputs
	)

static

Definition at line 1687 of file LinalgOps.cpp.

References buildGenericRegion(), and mlir::OpBuilder::create().

◆ buildStructuredOp()

static void buildStructuredOp	(	OpBuilder &	b,
		OperationState &	state,
		std::optional< TypeRange >	resultTensorTypes,
		ValueRange	inputs,
		ValueRange	outputs,
		ArrayRef< NamedAttribute >	attributes,
		RegionBuilderFn	regionBuilder
	)

static

Creates a structured operation given inputs, outputs, and attributes.

The result types are derived automatically if resultTensorTypes is none. The body of the operation is filled using regionBuilder. All ods-gen created structured operations use the method to implement their builders.

Definition at line 155 of file LinalgOps.cpp.

◆ buildSubAndExpOp()

static Value buildSubAndExpOp	(	OpBuilder &	builder,
		Location	loc,
		Value	input,
		Value	max,
		Value	output,
		int64_t	dim
	)

static

Produce a linalg generic that computes the second step of the softmax decomposition: res = exp(input - max), where max is the max of input on dimension dim.

Definition at line 2538 of file LinalgOps.cpp.

References computeIteratorTypesAndIndexingMaps(), mlir::OpBuilder::create(), mlir::Operation::getResult(), mlir::Value::getType(), and max().

◆ computeIteratorTypesAndIndexingMaps()

static std::tuple<SmallVector<utils::IteratorType>, SmallVector<AffineMap> > computeIteratorTypesAndIndexingMaps	(	OpBuilder &	builder,
		int64_t	inputRank,
		int64_t	dim,
		bool	allParallel = `false`
	)

static

Definition at line 2493 of file LinalgOps.cpp.

References mlir::AffineMap::get(), mlir::getAffineDimExpr(), mlir::Builder::getContext(), and mlir::AffineMap::getMultiDimIdentityMap().

Referenced by buildDivOp(), buildSubAndExpOp(), and reduce().

◆ fillStructuredOpRegion()

static void fillStructuredOpRegion	(	OpBuilder &	opBuilder,
		Region &	region,
		TypeRange	inputTypes,
		TypeRange	outputTypes,
		ArrayRef< NamedAttribute >	attrs,
		RegionBuilderFn	regionBuilder
	)

static

Fills the region of a structured operation using the provided regionBuilder.

The method is used by both named structured ops created by ods-gen and by manually defined C++ ops. It is called by both builders and parsers and creates a block with arguments corresponding to the elemental types of inputTypes and outputTypes. All output types are asserted to be ShapedType.

Definition at line 119 of file LinalgOps.cpp.

References mlir::OpBuilder::createBlock(), mlir::getElementTypeOrSelf(), mlir::Builder::getUnknownLoc(), and mlir::OpBuilder::setInsertionPointToStart().

Referenced by parseNamedStructuredOpRegion().

◆ findPayloadOp()

static Operation* findPayloadOp	(	Block *	body,
		bool	initFirst = `false`
	)

static

Definition at line 1326 of file LinalgOps.cpp.

References mlir::Block::getArgument(), mlir::Block::getArguments(), mlir::Block::getNumArguments(), mlir::Operation::getNumOperands(), mlir::Operation::getOperands(), and mlir::Block::getOperations().

◆ getDimValue()

static OpFoldResult getDimValue	(	OpBuilder &	builder,
		Location	loc,
		Value	v,
		int64_t	dim
	)

static

Return a memref.dim or tensor.dim for the shape of v at dim.

Definition at line 51 of file LinalgOps.cpp.

References mlir::getAsOpFoldResult(), mlir::Builder::getIndexAttr(), and mlir::Value::getType().

◆ getGenericEffectsImpl()

static void getGenericEffectsImpl	(	SmallVectorImpl< SideEffects::EffectInstance< MemoryEffects::Effect >> &	effects,
		ValueRange	results,
		const ValueRange	inputOperands,
		ValueRange	outputOperands
	)

static

Definition at line 1074 of file LinalgOps.cpp.

References mlir::SideEffects::Effect::Base< DerivedEffect, BaseEffect >::get(), mlir::SideEffects::Resource::Base< DefaultResource >::get(), and mlir::Value::getType().

◆ getSlice()

static Value getSlice	(	OpBuilder &	b,
		Location	loc,
		Value	source,
		ArrayRef< OpFoldResult >	offsets,
		ArrayRef< OpFoldResult >	sizes,
		ArrayRef< OpFoldResult >	strides
	)

static

Returns a memref.subview or a tensor.extract_slice based on the type of the source.

Definition at line 69 of file LinalgOps.cpp.

References mlir::OpBuilder::create(), and mlir::Value::getType().

◆ parseCommonStructuredOpParts()

static ParseResult parseCommonStructuredOpParts	(	OpAsmParser &	parser,
		OperationState &	result,
		SmallVectorImpl< Type > &	inputTypes,
		SmallVectorImpl< Type > &	outputTypes,
		bool	addOperandSegmentSizes = `true`
	)

static

Common parsing used for both named structured ops created by ods-gen and by manually defined C++ ops.

Does not handle regions.

Definition at line 185 of file LinalgOps.cpp.

Referenced by parseDstStyleOp(), and parseNamedStructuredOp().

◆ parseDenseI64ArrayAttr()

static ParseResult parseDenseI64ArrayAttr	(	OpAsmParser &	parser,
		NamedAttrList &	attributes,
		StringRef	attributeName
	)

static

Definition at line 1520 of file LinalgOps.cpp.

References mlir::failure(), mlir::detail::DenseArrayAttrImpl< T >::parse(), mlir::AsmParser::parseEqual(), mlir::AsmParser::parseKeyword(), mlir::NamedAttrList::set(), and mlir::success().

◆ parseDstStyleOp()

static ParseResult parseDstStyleOp	(	OpAsmParser &	parser,
		OperationState &	result,
		function_ref< ParseResult(OpAsmParser &, NamedAttrList &)>	parseAttrsFn = `nullptr`
	)

static

Definition at line 1199 of file LinalgOps.cpp.

References mlir::OperationState::addTypes(), mlir::OperationState::attributes, mlir::failed(), mlir::failure(), parseCommonStructuredOpParts(), mlir::AsmParser::parseOptionalAttrDict(), and mlir::success().

◆ parseNamedStructuredOp()

static ParseResult parseNamedStructuredOp	(	OpAsmParser &	parser,
		OperationState &	result,
		unsigned	numRegionArgs,
		RegionBuilderFn	regionBuilder
	)

static

Definition at line 297 of file LinalgOps.cpp.

References mlir::OperationState::addRegion(), mlir::OperationState::addTypes(), mlir::OperationState::attributes, mlir::failure(), mlir::NamedAttrList::getAttrs(), parseCommonStructuredOpParts(), parseNamedStructuredOpRegion(), parseNamedStructuredOpResults(), and mlir::success().

◆ parseNamedStructuredOpRegion()

static ParseResult parseNamedStructuredOpRegion	(	OpAsmParser &	parser,
		Region &	region,
		unsigned	numRegionArgs,
		TypeRange	inputTypes,
		TypeRange	outputTypes,
		ArrayRef< NamedAttribute >	attrs,
		RegionBuilderFn	regionBuilder
	)

static

Definition at line 271 of file LinalgOps.cpp.

References mlir::AsmParser::emitError(), fillStructuredOpRegion(), mlir::AsmParser::getContext(), mlir::AsmParser::getCurrentLocation(), and mlir::success().

Referenced by parseNamedStructuredOp().

◆ parseNamedStructuredOpResults()

static ParseResult parseNamedStructuredOpResults	(	OpAsmParser &	parser,
		SmallVectorImpl< Type > &	resultTypes
	)

static

Definition at line 290 of file LinalgOps.cpp.

References mlir::failure(), mlir::AsmParser::parseOptionalArrowTypeList(), and mlir::success().

Referenced by parseNamedStructuredOp().

◆ printCommonStructuredOpParts()

static void printCommonStructuredOpParts	(	OpAsmPrinter &	p,
		ValueRange	inputs,
		ValueRange	outputs
	)

static

Definition at line 259 of file LinalgOps.cpp.

References mlir::ValueRange::getTypes().

◆ printDenseI64ArrayAttr()

static void printDenseI64ArrayAttr	(	OpAsmPrinter &	p,
		StringRef	attributeName,
		ArrayRef< int64_t >	attributeValue
	)

static

Definition at line 1568 of file LinalgOps.cpp.

◆ printNamedStructuredOp()

static void printNamedStructuredOp	(	OpAsmPrinter &	p,
		Operation *	op,
		ValueRange	inputs,
		ValueRange	outputs
	)

static

Definition at line 330 of file LinalgOps.cpp.

◆ printNamedStructuredOpResults()

static void printNamedStructuredOpResults	(	OpAsmPrinter &	p,
		TypeRange	resultTypes
	)

static

Definition at line 323 of file LinalgOps.cpp.

References mlir::AsmPrinter::printOptionalArrowTypeList().

◆ printShortForm()

void printShortForm	(	OpAsmPrinter &	p,
		Operation *	payloadOp
	)

Definition at line 1355 of file LinalgOps.cpp.

References mlir::Operation::getAttrs(), mlir::Operation::getName(), mlir::OperationName::getStringRef(), and mlir::OpAsmPrinter::printOptionalAttrDict().

◆ reduce()

template<typename T >

static Value reduce	(	OpBuilder &	builder,
		Location	loc,
		Value	input,
		Value	output,
		int64_t	dim
	)

static

Definition at line 2515 of file LinalgOps.cpp.

References computeIteratorTypesAndIndexingMaps(), mlir::OpBuilder::create(), mlir::Operation::getResult(), and mlir::Value::getType().

Referenced by addAtomicRMW(), createDecl(), and declareReduction().

◆ verifyYield()

static LogicalResult verifyYield	(	linalg::YieldOp	op,
		LinalgOp	linalgOp
	)

static

Definition at line 1972 of file LinalgOps.cpp.

Macros

Typedefs

Functions

Macro Definition Documentation

◆ GET_OP_CLASSES [1/2]

◆ GET_OP_CLASSES [2/2]

Typedef Documentation

◆ RegionBuilderFn

Function Documentation

◆ addBodyWithPayloadOp()

◆ appendMangledType()

◆ buildDivOp()

◆ buildGenericRegion()

◆ buildIdentityRegion()

◆ buildStructuredOp()

◆ buildSubAndExpOp()

◆ computeIteratorTypesAndIndexingMaps()

◆ fillStructuredOpRegion()

◆ findPayloadOp()

◆ getDimValue()

◆ getGenericEffectsImpl()

◆ getSlice()

◆ parseCommonStructuredOpParts()

◆ parseDenseI64ArrayAttr()

◆ parseDstStyleOp()

◆ parseNamedStructuredOp()

◆ parseNamedStructuredOpRegion()

◆ parseNamedStructuredOpResults()

◆ printCommonStructuredOpParts()

◆ printDenseI64ArrayAttr()

◆ printNamedStructuredOp()

◆ printNamedStructuredOpResults()

◆ printShortForm()

◆ reduce()

◆ verifyYield()