VectorContractMultiLevelUnroll.cpp File Reference

#include "mlir/Dialect/SCF/IR/SCF.h"
#include "mlir/Dialect/Utils/StaticValueUtils.h"
#include "mlir/Dialect/Vector/IR/VectorOps.h"
#include "mlir/Dialect/Vector/Transforms/VectorRewritePatterns.h"
#include "mlir/Dialect/Vector/Utils/VectorUtils.h"
#include "mlir/Dialect/X86/Transforms.h"
#include "mlir/Dialect/X86/Utils/X86Utils.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/PatternMatch.h"
#include "mlir/Transforms/LoopInvariantCodeMotionUtils.h"

Go to the source code of this file.

Macros
#define	DEBUG_TYPE   "x86-vector-contract-multi-level-unroll"

Functions
static void	buildMNLoopBody (MLUCandidate &candidate, OpBuilder &b, Location loc, Value ivM, Value ivN)
static SmallVector< Value >	buildKLoopBody (MLUCandidate &candidate, OpBuilder &b, Location loc, Value ivM, Value ivN, Value ivK, ValueRange iterArgs)
static bool	isZeroVector (Value v)
static bool	isSimpleTransferOp (VectorTransferOpInterface op)
static bool	noMemoryEffectsBetween (Operation *start, Operation *end)
static Value	makeRankReducingSubview (OpBuilder &b, Location loc, Value base, ValueRange indices, VectorType targetType)
static SmallVector< Value >	slice (OpBuilder &b, Location loc, Value value, VectorType nativeType)
static Value	splice (OpBuilder &b, Location loc, ValueRange values, VectorType regTileType)
static std::optional< SmallVector< int64_t > >	getFromNativeShapeAttr (Operation *op)
static void	setNativeShapeAttr (VectorTransferOpInterface op, VectorType nativeType)
static LogicalResult	matchCanonicalMatmul (MLUCandidate &candidate, PatternRewriter &rewriter)
static LogicalResult	matchTypesAndShape (MLUCandidate &candidate, const MatchTypesAndShapeFn &matchTypesAndShapeForTarget, PatternRewriter &rewriter)
static LogicalResult	matchAccumulationLoop (MLUCandidate &candidate, PatternRewriter &rewriter)
static void	makeAccInitializationBackedByMemory (MLUCandidate &candidate, PatternRewriter &rewriter)
static void	makeAccResultBackedByMemory (MLUCandidate &candidate, PatternRewriter &rewriter)
static void	buildLoopNestForCandidate (MLUCandidate &candidate, PatternRewriter &rewriter)
static LogicalResult	matchTypesAndShapeForAMX (Type inputElemType, Type accumulatorElemType, bool isInVnniLayout, ArrayRef< int64_t > contractShape, SmallVectorImpl< int64_t > &contractRegTileShape, SmallVectorImpl< int64_t > &contractNativeShape)

Variables
static constexpr auto	nativeShapeAttrName = "x86_vcmlu_native_shape"

Macro Definition Documentation

DEBUG_TYPE

#define DEBUG_TYPE   "x86-vector-contract-multi-level-unroll"

Definition at line 21 of file VectorContractMultiLevelUnroll.cpp.

Function Documentation

buildKLoopBody()

SmallVector< Value > buildKLoopBody ( MLUCandidate & candidate, OpBuilder & b, Location loc, Value ivM, Value ivN, Value ivK, ValueRange iterArgs )

static

Definition at line 743 of file VectorContractMultiLevelUnroll.cpp.

References b, contract(), mlir::arith::ConstantIndexOp::create(), makeRankReducingSubview(), nativeShapeAttrName, setNativeShapeAttr(), slice(), and splice().

buildLoopNestForCandidate()

void buildLoopNestForCandidate ( MLUCandidate & candidate, PatternRewriter & rewriter )

static

Definition at line 641 of file VectorContractMultiLevelUnroll.cpp.

References b, buildMNLoopBody(), mlir::arith::ConstantIndexOp::create(), and mlir::OpBuilder::setInsertionPoint().

buildMNLoopBody()

void buildMNLoopBody ( MLUCandidate & candidate, OpBuilder & b, Location loc, Value ivM, Value ivN )

static

Definition at line 686 of file VectorContractMultiLevelUnroll.cpp.

References b, mlir::arith::ConstantIndexOp::create(), mlir::Operation::getResult(), setNativeShapeAttr(), slice(), and splice().

Referenced by buildLoopNestForCandidate().

getFromNativeShapeAttr()

std::optional< SmallVector< int64_t > > getFromNativeShapeAttr ( Operation * op )

static

Definition at line 236 of file VectorContractMultiLevelUnroll.cpp.

References mlir::Operation::getAttrOfType(), and nativeShapeAttrName.

Referenced by mlir::x86::populateVectorContractMultiLevelUnrollForAMXPatterns().

isSimpleTransferOp()

bool isSimpleTransferOp ( VectorTransferOpInterface op )

static

Definition at line 115 of file VectorContractMultiLevelUnroll.cpp.

Referenced by makeAccInitializationBackedByMemory(), makeAccResultBackedByMemory(), and matchAccumulationLoop().

isZeroVector()

bool isZeroVector ( Value v )

static

Definition at line 102 of file VectorContractMultiLevelUnroll.cpp.

References mlir::Value::getDefiningOp(), and mlir::isZeroIntegerOrFloat().

Referenced by matchAccumulationLoop().

makeAccInitializationBackedByMemory()

void makeAccInitializationBackedByMemory ( MLUCandidate & candidate, PatternRewriter & rewriter )

static

Definition at line 529 of file VectorContractMultiLevelUnroll.cpp.

References mlir::arith::ConstantIndexOp::create(), mlir::Value::getDefiningOp(), isSimpleTransferOp(), makeRankReducingSubview(), noMemoryEffectsBetween(), and mlir::OpBuilder::setInsertionPoint().

makeAccResultBackedByMemory()

void makeAccResultBackedByMemory ( MLUCandidate & candidate, PatternRewriter & rewriter )

static

Definition at line 569 of file VectorContractMultiLevelUnroll.cpp.

References mlir::arith::ConstantIndexOp::create(), mlir::getAsOpFoldResult(), mlir::Operation::getBlock(), mlir::Value::getDefiningOp(), mlir::Operation::isBeforeInBlock(), mlir::isEqualConstantIntOrValueArray(), isSimpleTransferOp(), makeRankReducingSubview(), noMemoryEffectsBetween(), mlir::RewriterBase::replaceAllUsesWith(), mlir::OpBuilder::setInsertionPoint(), and mlir::OpBuilder::setInsertionPointAfter().

makeRankReducingSubview()

Value makeRankReducingSubview ( OpBuilder & b, Location loc, Value base, ValueRange indices, VectorType targetType )

static

Definition at line 140 of file VectorContractMultiLevelUnroll.cpp.

References b, copy(), mlir::getAsIndexOpFoldResult(), mlir::getAsOpFoldResult(), mlir::Value::getType(), and indices.

Referenced by buildKLoopBody(), makeAccInitializationBackedByMemory(), and makeAccResultBackedByMemory().

matchAccumulationLoop()

LogicalResult matchAccumulationLoop ( MLUCandidate & candidate, PatternRewriter & rewriter )

static

Definition at line 448 of file VectorContractMultiLevelUnroll.cpp.

References contract(), mlir::getConstantIntValue(), mlir::BlockArgument::getOwner(), isSimpleTransferOp(), isZeroVector(), mlir::RewriterBase::notifyMatchFailure(), and success().

matchCanonicalMatmul()

LogicalResult matchCanonicalMatmul ( MLUCandidate & candidate, PatternRewriter & rewriter )

static

Definition at line 276 of file VectorContractMultiLevelUnroll.cpp.

References contract(), mlir::Builder::getAffineDimExpr(), mlir::x86::isInVnniLayout(), mlir::RewriterBase::notifyMatchFailure(), and success().

matchTypesAndShape()

LogicalResult matchTypesAndShape ( MLUCandidate & candidate, const MatchTypesAndShapeFn & matchTypesAndShapeForTarget, PatternRewriter & rewriter )

static

Definition at line 342 of file VectorContractMultiLevelUnroll.cpp.

References mlir::computeShapeRatio(), contract(), mlir::RewriterBase::notifyMatchFailure(), and success().

matchTypesAndShapeForAMX()

LogicalResult matchTypesAndShapeForAMX ( Type inputElemType, Type accumulatorElemType, bool isInVnniLayout, ArrayRef< int64_t > contractShape, SmallVectorImpl< int64_t > & contractRegTileShape, SmallVectorImpl< int64_t > & contractNativeShape )

static

Definition at line 854 of file VectorContractMultiLevelUnroll.cpp.

References mlir::Type::getIntOrFloatBitWidth(), mlir::Type::isBF16(), mlir::Type::isF32(), mlir::Type::isInteger(), and success().

Referenced by mlir::x86::populateVectorContractMultiLevelUnrollForAMXPatterns().

noMemoryEffectsBetween()

bool noMemoryEffectsBetween ( Operation * start, Operation * end )

static

Definition at line 122 of file VectorContractMultiLevelUnroll.cpp.

References mlir::Operation::getBlock(), mlir::Operation::isBeforeInBlock(), and mlir::isMemoryEffectFree().

Referenced by makeAccInitializationBackedByMemory(), and makeAccResultBackedByMemory().

setNativeShapeAttr()

void setNativeShapeAttr ( VectorTransferOpInterface op, VectorType nativeType )

static

Definition at line 243 of file VectorContractMultiLevelUnroll.cpp.

References mlir::detail::DenseArrayAttrImpl< int64_t >::get(), and nativeShapeAttrName.

Referenced by buildKLoopBody(), and buildMNLoopBody().

slice()

SmallVector< Value > slice ( OpBuilder & b, Location loc, Value value, VectorType nativeType )

static

Definition at line 169 of file VectorContractMultiLevelUnroll.cpp.

References b, mlir::Value::getType(), and result.

Referenced by blockArgsDominateInsertionPoint(), broadcastMatMulTSingletonDimension(), buildKLoopBody(), buildMNLoopBody(), mlir::affine::buildSliceTripCountMap(), checkAssumptionForLoop(), mlir::detail::defaultOperatesOnDisjointSubset(), mlir::detail::defaultOperatesOnEquivalentSubset(), dependsOnCarriedVals(), mlir::query::extractFunction(), mlir::xegpu::extractVectorsWithShapeFromValue(), getAdditionalComputeFraction(), getConsumerFromLoopUses(), mlir::affine::getFusionComputeCost(), mlir::Sliceable< PyOpResultList, PyOpResult >::getItemSlice(), mlir::presburger::detail::getNonOrthogonalVector(), mlir::getSlice(), mlir::AffineMap::getSliceMap(), mlir::memref::hasNoAliasingAccessInScope(), isFusionProfitable(), mlir::xegpu::lowerCrossLaneReductionToShuffles(), mlir::xegpu::lowerToVectorReductions(), mlir::moveOperationDependencies(), mlir::moveValueDefinitions(), mlir::presburger::IntMatrix::normalizeRow(), mlir::Diagnostic::print(), mlir::python::MLIR_BINDINGS_PYTHON_DOMAIN::PyOpOperands::PyOpOperands(), and mlir::tileAndFuseConsumer().

splice()

Value splice ( OpBuilder & b, Location loc, ValueRange values, VectorType regTileType )

static

Definition at line 197 of file VectorContractMultiLevelUnroll.cpp.

References b, mlir::ValueRange::getTypes(), and result.

Referenced by buildKLoopBody(), and buildMNLoopBody().

Variable Documentation

nativeShapeAttrName

auto nativeShapeAttrName = "x86_vcmlu_native_shape"

staticconstexpr

Definition at line 29 of file VectorContractMultiLevelUnroll.cpp.

Referenced by buildKLoopBody(), getFromNativeShapeAttr(), and setNativeShapeAttr().