MLIR  18.0.0git
GPUToSPIRV.cpp
Go to the documentation of this file.
1 //===- GPUToSPIRV.cpp - GPU to SPIR-V Patterns ----------------------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file implements patterns to convert GPU dialect to SPIR-V dialect.
10 //
11 //===----------------------------------------------------------------------===//
12 
21 #include "mlir/IR/BuiltinOps.h"
22 #include "mlir/IR/Matchers.h"
24 #include <optional>
25 
26 using namespace mlir;
27 
28 static constexpr const char kSPIRVModule[] = "__spv__";
29 
30 namespace {
31 /// Pattern lowering GPU block/thread size/id to loading SPIR-V invocation
32 /// builtin variables.
33 template <typename SourceOp, spirv::BuiltIn builtin>
34 class LaunchConfigConversion : public OpConversionPattern<SourceOp> {
35 public:
37 
39  matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
40  ConversionPatternRewriter &rewriter) const override;
41 };
42 
43 /// Pattern lowering subgroup size/id to loading SPIR-V invocation
44 /// builtin variables.
45 template <typename SourceOp, spirv::BuiltIn builtin>
46 class SingleDimLaunchConfigConversion : public OpConversionPattern<SourceOp> {
47 public:
49 
51  matchAndRewrite(SourceOp op, typename SourceOp::Adaptor adaptor,
52  ConversionPatternRewriter &rewriter) const override;
53 };
54 
55 /// This is separate because in Vulkan workgroup size is exposed to shaders via
56 /// a constant with WorkgroupSize decoration. So here we cannot generate a
57 /// builtin variable; instead the information in the `spirv.entry_point_abi`
58 /// attribute on the surrounding FuncOp is used to replace the gpu::BlockDimOp.
59 class WorkGroupSizeConversion : public OpConversionPattern<gpu::BlockDimOp> {
60 public:
61  WorkGroupSizeConversion(TypeConverter &typeConverter, MLIRContext *context)
62  : OpConversionPattern(typeConverter, context, /*benefit*/ 10) {}
63 
65  matchAndRewrite(gpu::BlockDimOp op, OpAdaptor adaptor,
66  ConversionPatternRewriter &rewriter) const override;
67 };
68 
69 /// Pattern to convert a kernel function in GPU dialect within a spirv.module.
70 class GPUFuncOpConversion final : public OpConversionPattern<gpu::GPUFuncOp> {
71 public:
73 
75  matchAndRewrite(gpu::GPUFuncOp funcOp, OpAdaptor adaptor,
76  ConversionPatternRewriter &rewriter) const override;
77 
78 private:
79  SmallVector<int32_t, 3> workGroupSizeAsInt32;
80 };
81 
82 /// Pattern to convert a gpu.module to a spirv.module.
83 class GPUModuleConversion final : public OpConversionPattern<gpu::GPUModuleOp> {
84 public:
86 
88  matchAndRewrite(gpu::GPUModuleOp moduleOp, OpAdaptor adaptor,
89  ConversionPatternRewriter &rewriter) const override;
90 };
91 
92 class GPUModuleEndConversion final
93  : public OpConversionPattern<gpu::ModuleEndOp> {
94 public:
96 
98  matchAndRewrite(gpu::ModuleEndOp endOp, OpAdaptor adaptor,
99  ConversionPatternRewriter &rewriter) const override {
100  rewriter.eraseOp(endOp);
101  return success();
102  }
103 };
104 
105 /// Pattern to convert a gpu.return into a SPIR-V return.
106 // TODO: This can go to DRR when GPU return has operands.
107 class GPUReturnOpConversion final : public OpConversionPattern<gpu::ReturnOp> {
108 public:
110 
112  matchAndRewrite(gpu::ReturnOp returnOp, OpAdaptor adaptor,
113  ConversionPatternRewriter &rewriter) const override;
114 };
115 
116 /// Pattern to convert a gpu.barrier op into a spirv.ControlBarrier op.
117 class GPUBarrierConversion final : public OpConversionPattern<gpu::BarrierOp> {
118 public:
120 
122  matchAndRewrite(gpu::BarrierOp barrierOp, OpAdaptor adaptor,
123  ConversionPatternRewriter &rewriter) const override;
124 };
125 
126 /// Pattern to convert a gpu.shuffle op into a spirv.GroupNonUniformShuffle op.
127 class GPUShuffleConversion final : public OpConversionPattern<gpu::ShuffleOp> {
128 public:
130 
132  matchAndRewrite(gpu::ShuffleOp shuffleOp, OpAdaptor adaptor,
133  ConversionPatternRewriter &rewriter) const override;
134 };
135 
136 } // namespace
137 
138 //===----------------------------------------------------------------------===//
139 // Builtins.
140 //===----------------------------------------------------------------------===//
141 
142 template <typename SourceOp, spirv::BuiltIn builtin>
143 LogicalResult LaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
144  SourceOp op, typename SourceOp::Adaptor adaptor,
145  ConversionPatternRewriter &rewriter) const {
146  auto *typeConverter = this->template getTypeConverter<SPIRVTypeConverter>();
147  Type indexType = typeConverter->getIndexType();
148 
149  // For Vulkan, these SPIR-V builtin variables are required to be a vector of
150  // type <3xi32> by the spec:
151  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/NumWorkgroups.html
152  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/WorkgroupId.html
153  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/WorkgroupSize.html
154  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/LocalInvocationId.html
155  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/LocalInvocationId.html
156  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/GlobalInvocationId.html
157  //
158  // For OpenCL, it depends on the Physical32/Physical64 addressing model:
159  // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_Env.html#_built_in_variables
160  bool forShader =
161  typeConverter->getTargetEnv().allows(spirv::Capability::Shader);
162  Type builtinType = forShader ? rewriter.getIntegerType(32) : indexType;
163 
164  Value vector =
165  spirv::getBuiltinVariableValue(op, builtin, builtinType, rewriter);
166  Value dim = rewriter.create<spirv::CompositeExtractOp>(
167  op.getLoc(), builtinType, vector,
168  rewriter.getI32ArrayAttr({static_cast<int32_t>(op.getDimension())}));
169  if (forShader && builtinType != indexType)
170  dim = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType, dim);
171  rewriter.replaceOp(op, dim);
172  return success();
173 }
174 
175 template <typename SourceOp, spirv::BuiltIn builtin>
177 SingleDimLaunchConfigConversion<SourceOp, builtin>::matchAndRewrite(
178  SourceOp op, typename SourceOp::Adaptor adaptor,
179  ConversionPatternRewriter &rewriter) const {
180  auto *typeConverter = this->template getTypeConverter<SPIRVTypeConverter>();
181  Type indexType = typeConverter->getIndexType();
182  Type i32Type = rewriter.getIntegerType(32);
183 
184  // For Vulkan, these SPIR-V builtin variables are required to be a vector of
185  // type i32 by the spec:
186  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/NumSubgroups.html
187  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/SubgroupId.html
188  // https://registry.khronos.org/vulkan/specs/1.3-extensions/man/html/SubgroupSize.html
189  //
190  // For OpenCL, they are also required to be i32:
191  // https://registry.khronos.org/OpenCL/specs/3.0-unified/html/OpenCL_Env.html#_built_in_variables
192  Value builtinValue =
193  spirv::getBuiltinVariableValue(op, builtin, i32Type, rewriter);
194  if (i32Type != indexType)
195  builtinValue = rewriter.create<spirv::UConvertOp>(op.getLoc(), indexType,
196  builtinValue);
197  rewriter.replaceOp(op, builtinValue);
198  return success();
199 }
200 
201 LogicalResult WorkGroupSizeConversion::matchAndRewrite(
202  gpu::BlockDimOp op, OpAdaptor adaptor,
203  ConversionPatternRewriter &rewriter) const {
204  DenseI32ArrayAttr workGroupSizeAttr = spirv::lookupLocalWorkGroupSize(op);
205  if (!workGroupSizeAttr)
206  return failure();
207 
208  int val =
209  workGroupSizeAttr.asArrayRef()[static_cast<int32_t>(op.getDimension())];
210  auto convertedType =
211  getTypeConverter()->convertType(op.getResult().getType());
212  if (!convertedType)
213  return failure();
214  rewriter.replaceOpWithNewOp<spirv::ConstantOp>(
215  op, convertedType, IntegerAttr::get(convertedType, val));
216  return success();
217 }
218 
219 //===----------------------------------------------------------------------===//
220 // GPUFuncOp
221 //===----------------------------------------------------------------------===//
222 
223 // Legalizes a GPU function as an entry SPIR-V function.
224 static spirv::FuncOp
225 lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter,
226  ConversionPatternRewriter &rewriter,
227  spirv::EntryPointABIAttr entryPointInfo,
229  auto fnType = funcOp.getFunctionType();
230  if (fnType.getNumResults()) {
231  funcOp.emitError("SPIR-V lowering only supports entry functions"
232  "with no return values right now");
233  return nullptr;
234  }
235  if (!argABIInfo.empty() && fnType.getNumInputs() != argABIInfo.size()) {
236  funcOp.emitError(
237  "lowering as entry functions requires ABI info for all arguments "
238  "or none of them");
239  return nullptr;
240  }
241  // Update the signature to valid SPIR-V types and add the ABI
242  // attributes. These will be "materialized" by using the
243  // LowerABIAttributesPass.
244  TypeConverter::SignatureConversion signatureConverter(fnType.getNumInputs());
245  {
246  for (const auto &argType :
247  enumerate(funcOp.getFunctionType().getInputs())) {
248  auto convertedType = typeConverter.convertType(argType.value());
249  if (!convertedType)
250  return nullptr;
251  signatureConverter.addInputs(argType.index(), convertedType);
252  }
253  }
254  auto newFuncOp = rewriter.create<spirv::FuncOp>(
255  funcOp.getLoc(), funcOp.getName(),
256  rewriter.getFunctionType(signatureConverter.getConvertedTypes(),
257  std::nullopt));
258  for (const auto &namedAttr : funcOp->getAttrs()) {
259  if (namedAttr.getName() == funcOp.getFunctionTypeAttrName() ||
260  namedAttr.getName() == SymbolTable::getSymbolAttrName())
261  continue;
262  newFuncOp->setAttr(namedAttr.getName(), namedAttr.getValue());
263  }
264 
265  rewriter.inlineRegionBefore(funcOp.getBody(), newFuncOp.getBody(),
266  newFuncOp.end());
267  if (failed(rewriter.convertRegionTypes(&newFuncOp.getBody(), typeConverter,
268  &signatureConverter)))
269  return nullptr;
270  rewriter.eraseOp(funcOp);
271 
272  // Set the attributes for argument and the function.
273  StringRef argABIAttrName = spirv::getInterfaceVarABIAttrName();
274  for (auto argIndex : llvm::seq<unsigned>(0, argABIInfo.size())) {
275  newFuncOp.setArgAttr(argIndex, argABIAttrName, argABIInfo[argIndex]);
276  }
277  newFuncOp->setAttr(spirv::getEntryPointABIAttrName(), entryPointInfo);
278 
279  return newFuncOp;
280 }
281 
282 /// Populates `argABI` with spirv.interface_var_abi attributes for lowering
283 /// gpu.func to spirv.func if no arguments have the attributes set
284 /// already. Returns failure if any argument has the ABI attribute set already.
285 static LogicalResult
286 getDefaultABIAttrs(const spirv::TargetEnv &targetEnv, gpu::GPUFuncOp funcOp,
288  if (!spirv::needsInterfaceVarABIAttrs(targetEnv))
289  return success();
290 
291  for (auto argIndex : llvm::seq<unsigned>(0, funcOp.getNumArguments())) {
292  if (funcOp.getArgAttrOfType<spirv::InterfaceVarABIAttr>(
294  return failure();
295  // Vulkan's interface variable requirements needs scalars to be wrapped in a
296  // struct. The struct held in storage buffer.
297  std::optional<spirv::StorageClass> sc;
298  if (funcOp.getArgument(argIndex).getType().isIntOrIndexOrFloat())
299  sc = spirv::StorageClass::StorageBuffer;
300  argABI.push_back(
301  spirv::getInterfaceVarABIAttr(0, argIndex, sc, funcOp.getContext()));
302  }
303  return success();
304 }
305 
306 LogicalResult GPUFuncOpConversion::matchAndRewrite(
307  gpu::GPUFuncOp funcOp, OpAdaptor adaptor,
308  ConversionPatternRewriter &rewriter) const {
309  if (!gpu::GPUDialect::isKernel(funcOp))
310  return failure();
311 
312  auto *typeConverter = getTypeConverter<SPIRVTypeConverter>();
314  if (failed(
315  getDefaultABIAttrs(typeConverter->getTargetEnv(), funcOp, argABI))) {
316  argABI.clear();
317  for (auto argIndex : llvm::seq<unsigned>(0, funcOp.getNumArguments())) {
318  // If the ABI is already specified, use it.
319  auto abiAttr = funcOp.getArgAttrOfType<spirv::InterfaceVarABIAttr>(
321  if (!abiAttr) {
322  funcOp.emitRemark(
323  "match failure: missing 'spirv.interface_var_abi' attribute at "
324  "argument ")
325  << argIndex;
326  return failure();
327  }
328  argABI.push_back(abiAttr);
329  }
330  }
331 
332  auto entryPointAttr = spirv::lookupEntryPointABI(funcOp);
333  if (!entryPointAttr) {
334  funcOp.emitRemark(
335  "match failure: missing 'spirv.entry_point_abi' attribute");
336  return failure();
337  }
338  spirv::FuncOp newFuncOp = lowerAsEntryFunction(
339  funcOp, *getTypeConverter(), rewriter, entryPointAttr, argABI);
340  if (!newFuncOp)
341  return failure();
342  newFuncOp->removeAttr(
343  rewriter.getStringAttr(gpu::GPUDialect::getKernelFuncAttrName()));
344  return success();
345 }
346 
347 //===----------------------------------------------------------------------===//
348 // ModuleOp with gpu.module.
349 //===----------------------------------------------------------------------===//
350 
351 LogicalResult GPUModuleConversion::matchAndRewrite(
352  gpu::GPUModuleOp moduleOp, OpAdaptor adaptor,
353  ConversionPatternRewriter &rewriter) const {
354  auto *typeConverter = getTypeConverter<SPIRVTypeConverter>();
355  const spirv::TargetEnv &targetEnv = typeConverter->getTargetEnv();
356  spirv::AddressingModel addressingModel = spirv::getAddressingModel(
357  targetEnv, typeConverter->getOptions().use64bitIndex);
358  FailureOr<spirv::MemoryModel> memoryModel = spirv::getMemoryModel(targetEnv);
359  if (failed(memoryModel))
360  return moduleOp.emitRemark(
361  "cannot deduce memory model from 'spirv.target_env'");
362 
363  // Add a keyword to the module name to avoid symbolic conflict.
364  std::string spvModuleName = (kSPIRVModule + moduleOp.getName()).str();
365  auto spvModule = rewriter.create<spirv::ModuleOp>(
366  moduleOp.getLoc(), addressingModel, *memoryModel, std::nullopt,
367  StringRef(spvModuleName));
368 
369  // Move the region from the module op into the SPIR-V module.
370  Region &spvModuleRegion = spvModule.getRegion();
371  rewriter.inlineRegionBefore(moduleOp.getBodyRegion(), spvModuleRegion,
372  spvModuleRegion.begin());
373  // The spirv.module build method adds a block. Remove that.
374  rewriter.eraseBlock(&spvModuleRegion.back());
375 
376  // Some of the patterns call `lookupTargetEnv` during conversion and they
377  // will fail if called after GPUModuleConversion and we don't preserve
378  // `TargetEnv` attribute.
379  // Copy TargetEnvAttr only if it is attached directly to the GPUModuleOp.
380  if (auto attr = moduleOp->getAttrOfType<spirv::TargetEnvAttr>(
382  spvModule->setAttr(spirv::getTargetEnvAttrName(), attr);
383 
384  rewriter.eraseOp(moduleOp);
385  return success();
386 }
387 
388 //===----------------------------------------------------------------------===//
389 // GPU return inside kernel functions to SPIR-V return.
390 //===----------------------------------------------------------------------===//
391 
392 LogicalResult GPUReturnOpConversion::matchAndRewrite(
393  gpu::ReturnOp returnOp, OpAdaptor adaptor,
394  ConversionPatternRewriter &rewriter) const {
395  if (!adaptor.getOperands().empty())
396  return failure();
397 
398  rewriter.replaceOpWithNewOp<spirv::ReturnOp>(returnOp);
399  return success();
400 }
401 
402 //===----------------------------------------------------------------------===//
403 // Barrier.
404 //===----------------------------------------------------------------------===//
405 
406 LogicalResult GPUBarrierConversion::matchAndRewrite(
407  gpu::BarrierOp barrierOp, OpAdaptor adaptor,
408  ConversionPatternRewriter &rewriter) const {
409  MLIRContext *context = getContext();
410  // Both execution and memory scope should be workgroup.
411  auto scope = spirv::ScopeAttr::get(context, spirv::Scope::Workgroup);
412  // Require acquire and release memory semantics for workgroup memory.
413  auto memorySemantics = spirv::MemorySemanticsAttr::get(
414  context, spirv::MemorySemantics::WorkgroupMemory |
415  spirv::MemorySemantics::AcquireRelease);
416  rewriter.replaceOpWithNewOp<spirv::ControlBarrierOp>(barrierOp, scope, scope,
417  memorySemantics);
418  return success();
419 }
420 
421 //===----------------------------------------------------------------------===//
422 // Shuffle
423 //===----------------------------------------------------------------------===//
424 
425 LogicalResult GPUShuffleConversion::matchAndRewrite(
426  gpu::ShuffleOp shuffleOp, OpAdaptor adaptor,
427  ConversionPatternRewriter &rewriter) const {
428  // Require the shuffle width to be the same as the target's subgroup size,
429  // given that for SPIR-V non-uniform subgroup ops, we cannot select
430  // participating invocations.
431  auto targetEnv = getTypeConverter<SPIRVTypeConverter>()->getTargetEnv();
432  unsigned subgroupSize =
433  targetEnv.getAttr().getResourceLimits().getSubgroupSize();
434  IntegerAttr widthAttr;
435  if (!matchPattern(shuffleOp.getWidth(), m_Constant(&widthAttr)) ||
436  widthAttr.getValue().getZExtValue() != subgroupSize)
437  return rewriter.notifyMatchFailure(
438  shuffleOp, "shuffle width and target subgroup size mismatch");
439 
440  Location loc = shuffleOp.getLoc();
441  Value trueVal = spirv::ConstantOp::getOne(rewriter.getI1Type(),
442  shuffleOp.getLoc(), rewriter);
443  auto scope = rewriter.getAttr<spirv::ScopeAttr>(spirv::Scope::Subgroup);
444  Value result;
445 
446  switch (shuffleOp.getMode()) {
447  case gpu::ShuffleMode::XOR:
448  result = rewriter.create<spirv::GroupNonUniformShuffleXorOp>(
449  loc, scope, adaptor.getValue(), adaptor.getOffset());
450  break;
451  case gpu::ShuffleMode::IDX:
452  result = rewriter.create<spirv::GroupNonUniformShuffleOp>(
453  loc, scope, adaptor.getValue(), adaptor.getOffset());
454  break;
455  default:
456  return rewriter.notifyMatchFailure(shuffleOp, "unimplemented shuffle mode");
457  }
458 
459  rewriter.replaceOp(shuffleOp, {result, trueVal});
460  return success();
461 }
462 
463 //===----------------------------------------------------------------------===//
464 // Group ops
465 //===----------------------------------------------------------------------===//
466 
467 template <typename UniformOp, typename NonUniformOp>
469  Value arg, bool isGroup, bool isUniform) {
470  Type type = arg.getType();
471  auto scope = mlir::spirv::ScopeAttr::get(builder.getContext(),
472  isGroup ? spirv::Scope::Workgroup
473  : spirv::Scope::Subgroup);
474  auto groupOp = spirv::GroupOperationAttr::get(builder.getContext(),
475  spirv::GroupOperation::Reduce);
476  if (isUniform) {
477  return builder.create<UniformOp>(loc, type, scope, groupOp, arg)
478  .getResult();
479  }
480  return builder.create<NonUniformOp>(loc, type, scope, groupOp, arg, Value{})
481  .getResult();
482 }
483 
484 static std::optional<Value> createGroupReduceOp(OpBuilder &builder,
485  Location loc, Value arg,
486  gpu::AllReduceOperation opType,
487  bool isGroup, bool isUniform) {
488  using FuncT = Value (*)(OpBuilder &, Location, Value, bool, bool);
489  struct OpHandler {
490  gpu::AllReduceOperation type;
491  FuncT intFunc;
492  FuncT floatFunc;
493  };
494 
495  Type type = arg.getType();
496  using MembptrT = FuncT OpHandler::*;
497  MembptrT handlerPtr;
498  if (isa<FloatType>(type)) {
499  handlerPtr = &OpHandler::floatFunc;
500  } else if (isa<IntegerType>(type)) {
501  handlerPtr = &OpHandler::intFunc;
502  } else {
503  return std::nullopt;
504  }
505 
506  using ReduceType = gpu::AllReduceOperation;
507  namespace spv = spirv;
508  const OpHandler handlers[] = {
509  {ReduceType::ADD,
510  &createGroupReduceOpImpl<spv::GroupIAddOp, spv::GroupNonUniformIAddOp>,
511  &createGroupReduceOpImpl<spv::GroupFAddOp, spv::GroupNonUniformFAddOp>},
512  {ReduceType::MUL,
513  &createGroupReduceOpImpl<spv::GroupIMulKHROp,
514  spv::GroupNonUniformIMulOp>,
515  &createGroupReduceOpImpl<spv::GroupFMulKHROp,
516  spv::GroupNonUniformFMulOp>},
517  {ReduceType::MIN,
518  &createGroupReduceOpImpl<spv::GroupSMinOp, spv::GroupNonUniformSMinOp>,
519  &createGroupReduceOpImpl<spv::GroupFMinOp, spv::GroupNonUniformFMinOp>},
520  {ReduceType::MAX,
521  &createGroupReduceOpImpl<spv::GroupSMaxOp, spv::GroupNonUniformSMaxOp>,
522  &createGroupReduceOpImpl<spv::GroupFMaxOp, spv::GroupNonUniformFMaxOp>},
523  };
524 
525  for (auto &handler : handlers)
526  if (handler.type == opType)
527  return (handler.*handlerPtr)(builder, loc, arg, isGroup, isUniform);
528 
529  return std::nullopt;
530 }
531 
532 /// Pattern to convert a gpu.all_reduce op into a SPIR-V group op.
534  : public OpConversionPattern<gpu::AllReduceOp> {
535 public:
537 
539  matchAndRewrite(gpu::AllReduceOp op, OpAdaptor adaptor,
540  ConversionPatternRewriter &rewriter) const override {
541  auto opType = op.getOp();
542 
543  // gpu.all_reduce can have either reduction op attribute or reduction
544  // region. Only attribute version is supported.
545  if (!opType)
546  return failure();
547 
548  auto result =
549  createGroupReduceOp(rewriter, op.getLoc(), adaptor.getValue(), *opType,
550  /*isGroup*/ true, op.getUniform());
551  if (!result)
552  return failure();
553 
554  rewriter.replaceOp(op, *result);
555  return success();
556  }
557 };
558 
559 /// Pattern to convert a gpu.subgroup_reduce op into a SPIR-V group op.
561  : public OpConversionPattern<gpu::SubgroupReduceOp> {
562 public:
564 
566  matchAndRewrite(gpu::SubgroupReduceOp op, OpAdaptor adaptor,
567  ConversionPatternRewriter &rewriter) const override {
568  auto opType = op.getOp();
569  auto result =
570  createGroupReduceOp(rewriter, op.getLoc(), adaptor.getValue(), opType,
571  /*isGroup*/ false, op.getUniform());
572  if (!result)
573  return failure();
574 
575  rewriter.replaceOp(op, *result);
576  return success();
577  }
578 };
579 
580 //===----------------------------------------------------------------------===//
581 // GPU To SPIRV Patterns.
582 //===----------------------------------------------------------------------===//
583 
585  RewritePatternSet &patterns) {
586  patterns.add<
587  GPUBarrierConversion, GPUFuncOpConversion, GPUModuleConversion,
588  GPUModuleEndConversion, GPUReturnOpConversion, GPUShuffleConversion,
589  LaunchConfigConversion<gpu::BlockIdOp, spirv::BuiltIn::WorkgroupId>,
590  LaunchConfigConversion<gpu::GridDimOp, spirv::BuiltIn::NumWorkgroups>,
591  LaunchConfigConversion<gpu::BlockDimOp, spirv::BuiltIn::WorkgroupSize>,
592  LaunchConfigConversion<gpu::ThreadIdOp,
593  spirv::BuiltIn::LocalInvocationId>,
594  LaunchConfigConversion<gpu::GlobalIdOp,
595  spirv::BuiltIn::GlobalInvocationId>,
596  SingleDimLaunchConfigConversion<gpu::SubgroupIdOp,
597  spirv::BuiltIn::SubgroupId>,
598  SingleDimLaunchConfigConversion<gpu::NumSubgroupsOp,
599  spirv::BuiltIn::NumSubgroups>,
600  SingleDimLaunchConfigConversion<gpu::SubgroupSizeOp,
601  spirv::BuiltIn::SubgroupSize>,
602  WorkGroupSizeConversion, GPUAllReduceConversion,
603  GPUSubgroupReduceConversion>(typeConverter, patterns.getContext());
604 }
static Value createGroupReduceOpImpl(OpBuilder &builder, Location loc, Value arg, bool isGroup, bool isUniform)
Definition: GPUToSPIRV.cpp:468
static LogicalResult getDefaultABIAttrs(const spirv::TargetEnv &targetEnv, gpu::GPUFuncOp funcOp, SmallVectorImpl< spirv::InterfaceVarABIAttr > &argABI)
Populates argABI with spirv.interface_var_abi attributes for lowering gpu.func to spirv....
Definition: GPUToSPIRV.cpp:286
static std::optional< Value > createGroupReduceOp(OpBuilder &builder, Location loc, Value arg, gpu::AllReduceOperation opType, bool isGroup, bool isUniform)
Definition: GPUToSPIRV.cpp:484
static constexpr const char kSPIRVModule[]
Definition: GPUToSPIRV.cpp:28
static spirv::FuncOp lowerAsEntryFunction(gpu::GPUFuncOp funcOp, const TypeConverter &typeConverter, ConversionPatternRewriter &rewriter, spirv::EntryPointABIAttr entryPointInfo, ArrayRef< spirv::InterfaceVarABIAttr > argABIInfo)
Definition: GPUToSPIRV.cpp:225
static MLIRContext * getContext(OpFoldResult val)
Pattern to convert a gpu.all_reduce op into a SPIR-V group op.
Definition: GPUToSPIRV.cpp:534
LogicalResult matchAndRewrite(gpu::AllReduceOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Definition: GPUToSPIRV.cpp:539
Pattern to convert a gpu.subgroup_reduce op into a SPIR-V group op.
Definition: GPUToSPIRV.cpp:561
LogicalResult matchAndRewrite(gpu::SubgroupReduceOp op, OpAdaptor adaptor, ConversionPatternRewriter &rewriter) const override
Definition: GPUToSPIRV.cpp:566
ArrayAttr getI32ArrayAttr(ArrayRef< int32_t > values)
Definition: Builders.cpp:283
FunctionType getFunctionType(TypeRange inputs, TypeRange results)
Definition: Builders.cpp:96
IntegerType getIntegerType(unsigned width)
Definition: Builders.cpp:87
StringAttr getStringAttr(const Twine &bytes)
Definition: Builders.cpp:269
MLIRContext * getContext() const
Definition: Builders.h:55
IntegerType getI1Type()
Definition: Builders.cpp:73
Attr getAttr(Args &&...args)
Get or construct an instance of the attribute Attr with provided arguments.
Definition: Builders.h:100
This class implements a pattern rewriter for use with ConversionPatterns.
void replaceOp(Operation *op, ValueRange newValues) override
PatternRewriter hook for replacing an operation.
FailureOr< Block * > convertRegionTypes(Region *region, const TypeConverter &converter, TypeConverter::SignatureConversion *entryConversion=nullptr)
Convert the types of block arguments within the given region.
LogicalResult notifyMatchFailure(Location loc, function_ref< void(Diagnostic &)> reasonCallback) override
PatternRewriter hook for notifying match failure reasons.
void inlineRegionBefore(Region &region, Region &parent, Region::iterator before) override
PatternRewriter hook for moving blocks out of a region.
void eraseOp(Operation *op) override
PatternRewriter hook for erasing a dead operation.
void eraseBlock(Block *block) override
PatternRewriter hook for erase all operations in a block.
This class provides support for representing a failure result, or a valid value of type T.
Definition: LogicalResult.h:78
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:63
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:60
This class helps build Operations.
Definition: Builders.h:206
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:446
OpConversionPattern is a wrapper around ConversionPattern that allows for matching and rewriting agai...
OpConversionPattern(MLIRContext *context, PatternBenefit benefit=1)
OpResult getResult(unsigned idx)
Get the 'idx'th result of this operation.
Definition: Operation.h:402
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
void setAttr(StringAttr name, Attribute value)
If the an attribute exists with the specified name, change it to the new value.
Definition: Operation.h:560
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
MLIRContext * getContext() const
RewritePatternSet & add(ConstructorArg &&arg, ConstructorArgs &&...args)
Add an instance of each of the pattern types 'Ts' to the pattern list with the given arguments.
OpTy replaceOpWithNewOp(Operation *op, Args &&...args)
Replaces the result op with a new op that is created without verification.
Definition: PatternMatch.h:539
Type conversion from builtin types to SPIR-V types for shader interface.
static StringRef getSymbolAttrName()
Return the name of the attribute used for symbol names.
Definition: SymbolTable.h:59
This class provides all of the information necessary to convert a type signature.
void addInputs(unsigned origInputNo, ArrayRef< Type > types)
Remap an input of the original signature with a new set of types.
ArrayRef< Type > getConvertedTypes() const
Return the argument types for the new signature.
Type conversion class.
LogicalResult convertType(Type t, SmallVectorImpl< Type > &results) const
Convert the given type.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:93
Type getType() const
Return the type of this value.
Definition: Value.h:122
An attribute that specifies the information regarding the interface variable: descriptor set,...
An attribute that specifies the target version, allowed extensions and capabilities,...
ResourceLimitsAttr getResourceLimits() const
Returns the target resource limits.
A wrapper class around a spirv::TargetEnvAttr to provide query methods for allowed version/capabiliti...
Definition: TargetAndABI.h:29
TargetEnvAttr getAttr() const
Definition: TargetAndABI.h:62
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:285
StringRef getInterfaceVarABIAttrName()
Returns the attribute name for specifying argument ABI information.
bool needsInterfaceVarABIAttrs(TargetEnvAttr targetAttr)
Returns whether the given SPIR-V target (described by TargetEnvAttr) needs ABI attributes for interfa...
InterfaceVarABIAttr getInterfaceVarABIAttr(unsigned descriptorSet, unsigned binding, std::optional< StorageClass > storageClass, MLIRContext *context)
Gets the InterfaceVarABIAttr given its fields.
Value getBuiltinVariableValue(Operation *op, BuiltIn builtin, Type integerType, OpBuilder &builder, StringRef prefix="__builtin__", StringRef suffix="__")
Returns the value for the given builtin variable.
EntryPointABIAttr lookupEntryPointABI(Operation *op)
Queries the entry point ABI on the nearest function-like op containing the given op.
StringRef getTargetEnvAttrName()
Returns the attribute name for specifying SPIR-V target environment.
DenseI32ArrayAttr lookupLocalWorkGroupSize(Operation *op)
Queries the local workgroup size from entry point ABI on the nearest function-like op containing the ...
AddressingModel getAddressingModel(TargetEnvAttr targetAttr, bool use64bitAddress)
Returns addressing model selected based on target environment.
FailureOr< MemoryModel > getMemoryModel(TargetEnvAttr targetAttr)
Returns memory model selected based on target environment.
StringRef getEntryPointABIAttrName()
Returns the attribute name for specifying entry point information.
This header declares functions that assist transformations in the MemRef dialect.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:401
LogicalResult failure(bool isFailure=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:62
LogicalResult success(bool isSuccess=true)
Utility function to generate a LogicalResult.
Definition: LogicalResult.h:56
void populateGPUToSPIRVPatterns(SPIRVTypeConverter &typeConverter, RewritePatternSet &patterns)
Appends to a pattern list additional patterns for translating GPU Ops to SPIR-V ops.
Definition: GPUToSPIRV.cpp:584
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Definition: Matchers.h:310
bool failed(LogicalResult result)
Utility function that returns true if the provided LogicalResult corresponds to a failure value.
Definition: LogicalResult.h:72
This class represents an efficient way to signal success or failure.
Definition: LogicalResult.h:26