MLIR  21.0.0git
LLVMDialect.cpp
Go to the documentation of this file.
1 //===- LLVMDialect.cpp - LLVM IR Ops and Dialect registration -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the types and operation details for the LLVM IR dialect in
10 // MLIR, and the LLVM IR dialect. It also registers the dialect.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "TypeDetail.h"
19 #include "mlir/IR/Attributes.h"
20 #include "mlir/IR/Builders.h"
21 #include "mlir/IR/BuiltinOps.h"
22 #include "mlir/IR/BuiltinTypes.h"
24 #include "mlir/IR/MLIRContext.h"
25 #include "mlir/IR/Matchers.h"
28 
29 #include "llvm/ADT/SCCIterator.h"
30 #include "llvm/ADT/TypeSwitch.h"
31 #include "llvm/AsmParser/Parser.h"
32 #include "llvm/Bitcode/BitcodeReader.h"
33 #include "llvm/Bitcode/BitcodeWriter.h"
34 #include "llvm/IR/Attributes.h"
35 #include "llvm/IR/Function.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/Support/Error.h"
38 #include "llvm/Support/Mutex.h"
39 #include "llvm/Support/SourceMgr.h"
40 
41 #include <numeric>
42 #include <optional>
43 
44 using namespace mlir;
45 using namespace mlir::LLVM;
46 using mlir::LLVM::cconv::getMaxEnumValForCConv;
47 using mlir::LLVM::linkage::getMaxEnumValForLinkage;
48 using mlir::LLVM::tailcallkind::getMaxEnumValForTailCallKind;
49 
50 #include "mlir/Dialect/LLVMIR/LLVMOpsDialect.cpp.inc"
51 
52 //===----------------------------------------------------------------------===//
53 // Attribute Helpers
54 //===----------------------------------------------------------------------===//
55 
56 static constexpr const char kElemTypeAttrName[] = "elem_type";
57 
59  SmallVector<NamedAttribute, 8> filteredAttrs(
60  llvm::make_filter_range(attrs, [&](NamedAttribute attr) {
61  if (attr.getName() == "fastmathFlags") {
62  auto defAttr =
63  FastmathFlagsAttr::get(attr.getValue().getContext(), {});
64  return defAttr != attr.getValue();
65  }
66  return true;
67  }));
68  return filteredAttrs;
69 }
70 
71 /// Verifies `symbol`'s use in `op` to ensure the symbol is a valid and
72 /// fully defined llvm.func.
73 static LogicalResult verifySymbolAttrUse(FlatSymbolRefAttr symbol,
74  Operation *op,
75  SymbolTableCollection &symbolTable) {
76  StringRef name = symbol.getValue();
77  auto func =
78  symbolTable.lookupNearestSymbolFrom<LLVMFuncOp>(op, symbol.getAttr());
79  if (!func)
80  return op->emitOpError("'")
81  << name << "' does not reference a valid LLVM function";
82  if (func.isExternal())
83  return op->emitOpError("'") << name << "' does not have a definition";
84  return success();
85 }
86 
87 /// Returns a boolean type that has the same shape as `type`. It supports both
88 /// fixed size vectors as well as scalable vectors.
89 static Type getI1SameShape(Type type) {
90  Type i1Type = IntegerType::get(type.getContext(), 1);
92  return LLVM::getVectorType(i1Type, LLVM::getVectorNumElements(type));
93  return i1Type;
94 }
95 
96 // Parses one of the keywords provided in the list `keywords` and returns the
97 // position of the parsed keyword in the list. If none of the keywords from the
98 // list is parsed, returns -1.
100  ArrayRef<StringRef> keywords) {
101  for (const auto &en : llvm::enumerate(keywords)) {
102  if (succeeded(parser.parseOptionalKeyword(en.value())))
103  return en.index();
104  }
105  return -1;
106 }
107 
108 namespace {
109 template <typename Ty>
110 struct EnumTraits {};
111 
112 #define REGISTER_ENUM_TYPE(Ty) \
113  template <> \
114  struct EnumTraits<Ty> { \
115  static StringRef stringify(Ty value) { return stringify##Ty(value); } \
116  static unsigned getMaxEnumVal() { return getMaxEnumValFor##Ty(); } \
117  }
118 
119 REGISTER_ENUM_TYPE(Linkage);
120 REGISTER_ENUM_TYPE(UnnamedAddr);
121 REGISTER_ENUM_TYPE(CConv);
122 REGISTER_ENUM_TYPE(TailCallKind);
124 } // namespace
125 
126 /// Parse an enum from the keyword, or default to the provided default value.
127 /// The return type is the enum type by default, unless overridden with the
128 /// second template argument.
129 template <typename EnumTy, typename RetTy = EnumTy>
131  OperationState &result,
132  EnumTy defaultValue) {
134  for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
135  names.push_back(EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i)));
136 
137  int index = parseOptionalKeywordAlternative(parser, names);
138  if (index == -1)
139  return static_cast<RetTy>(defaultValue);
140  return static_cast<RetTy>(index);
141 }
142 
143 //===----------------------------------------------------------------------===//
144 // Operand bundle helpers.
145 //===----------------------------------------------------------------------===//
146 
147 static void printOneOpBundle(OpAsmPrinter &p, OperandRange operands,
148  TypeRange operandTypes, StringRef tag) {
149  p.printString(tag);
150  p << "(";
151 
152  if (!operands.empty()) {
153  p.printOperands(operands);
154  p << " : ";
155  llvm::interleaveComma(operandTypes, p);
156  }
157 
158  p << ")";
159 }
160 
162  OperandRangeRange opBundleOperands,
163  TypeRangeRange opBundleOperandTypes,
164  std::optional<ArrayAttr> opBundleTags) {
165  if (opBundleOperands.empty())
166  return;
167  assert(opBundleTags && "expect operand bundle tags");
168 
169  p << "[";
170  llvm::interleaveComma(
171  llvm::zip(opBundleOperands, opBundleOperandTypes, *opBundleTags), p,
172  [&p](auto bundle) {
173  auto bundleTag = cast<StringAttr>(std::get<2>(bundle)).getValue();
174  printOneOpBundle(p, std::get<0>(bundle), std::get<1>(bundle),
175  bundleTag);
176  });
177  p << "]";
178 }
179 
180 static ParseResult parseOneOpBundle(
181  OpAsmParser &p,
183  SmallVector<SmallVector<Type>> &opBundleOperandTypes,
184  SmallVector<Attribute> &opBundleTags) {
185  SMLoc currentParserLoc = p.getCurrentLocation();
187  SmallVector<Type> types;
188  std::string tag;
189 
190  if (p.parseString(&tag))
191  return p.emitError(currentParserLoc, "expect operand bundle tag");
192 
193  if (p.parseLParen())
194  return failure();
195 
196  if (p.parseOptionalRParen()) {
197  if (p.parseOperandList(operands) || p.parseColon() ||
198  p.parseTypeList(types) || p.parseRParen())
199  return failure();
200  }
201 
202  opBundleOperands.push_back(std::move(operands));
203  opBundleOperandTypes.push_back(std::move(types));
204  opBundleTags.push_back(StringAttr::get(p.getContext(), tag));
205 
206  return success();
207 }
208 
209 static std::optional<ParseResult> parseOpBundles(
210  OpAsmParser &p,
212  SmallVector<SmallVector<Type>> &opBundleOperandTypes,
213  ArrayAttr &opBundleTags) {
214  if (p.parseOptionalLSquare())
215  return std::nullopt;
216 
217  if (succeeded(p.parseOptionalRSquare()))
218  return success();
219 
220  SmallVector<Attribute> opBundleTagAttrs;
221  auto bundleParser = [&] {
222  return parseOneOpBundle(p, opBundleOperands, opBundleOperandTypes,
223  opBundleTagAttrs);
224  };
225  if (p.parseCommaSeparatedList(bundleParser))
226  return failure();
227 
228  if (p.parseRSquare())
229  return failure();
230 
231  opBundleTags = ArrayAttr::get(p.getContext(), opBundleTagAttrs);
232 
233  return success();
234 }
235 
236 //===----------------------------------------------------------------------===//
237 // Printing, parsing, folding and builder for LLVM::CmpOp.
238 //===----------------------------------------------------------------------===//
239 
240 void ICmpOp::print(OpAsmPrinter &p) {
241  p << " \"" << stringifyICmpPredicate(getPredicate()) << "\" " << getOperand(0)
242  << ", " << getOperand(1);
243  p.printOptionalAttrDict((*this)->getAttrs(), {"predicate"});
244  p << " : " << getLhs().getType();
245 }
246 
247 void FCmpOp::print(OpAsmPrinter &p) {
248  p << " \"" << stringifyFCmpPredicate(getPredicate()) << "\" " << getOperand(0)
249  << ", " << getOperand(1);
250  p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()), {"predicate"});
251  p << " : " << getLhs().getType();
252 }
253 
254 // <operation> ::= `llvm.icmp` string-literal ssa-use `,` ssa-use
255 // attribute-dict? `:` type
256 // <operation> ::= `llvm.fcmp` string-literal ssa-use `,` ssa-use
257 // attribute-dict? `:` type
258 template <typename CmpPredicateType>
259 static ParseResult parseCmpOp(OpAsmParser &parser, OperationState &result) {
260  StringAttr predicateAttr;
262  Type type;
263  SMLoc predicateLoc, trailingTypeLoc;
264  if (parser.getCurrentLocation(&predicateLoc) ||
265  parser.parseAttribute(predicateAttr, "predicate", result.attributes) ||
266  parser.parseOperand(lhs) || parser.parseComma() ||
267  parser.parseOperand(rhs) ||
268  parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
269  parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type) ||
270  parser.resolveOperand(lhs, type, result.operands) ||
271  parser.resolveOperand(rhs, type, result.operands))
272  return failure();
273 
274  // Replace the string attribute `predicate` with an integer attribute.
275  int64_t predicateValue = 0;
276  if (std::is_same<CmpPredicateType, ICmpPredicate>()) {
277  std::optional<ICmpPredicate> predicate =
278  symbolizeICmpPredicate(predicateAttr.getValue());
279  if (!predicate)
280  return parser.emitError(predicateLoc)
281  << "'" << predicateAttr.getValue()
282  << "' is an incorrect value of the 'predicate' attribute";
283  predicateValue = static_cast<int64_t>(*predicate);
284  } else {
285  std::optional<FCmpPredicate> predicate =
286  symbolizeFCmpPredicate(predicateAttr.getValue());
287  if (!predicate)
288  return parser.emitError(predicateLoc)
289  << "'" << predicateAttr.getValue()
290  << "' is an incorrect value of the 'predicate' attribute";
291  predicateValue = static_cast<int64_t>(*predicate);
292  }
293 
294  result.attributes.set("predicate",
295  parser.getBuilder().getI64IntegerAttr(predicateValue));
296 
297  // The result type is either i1 or a vector type <? x i1> if the inputs are
298  // vectors.
299  if (!isCompatibleType(type))
300  return parser.emitError(trailingTypeLoc,
301  "expected LLVM dialect-compatible type");
302  result.addTypes(getI1SameShape(type));
303  return success();
304 }
305 
306 ParseResult ICmpOp::parse(OpAsmParser &parser, OperationState &result) {
307  return parseCmpOp<ICmpPredicate>(parser, result);
308 }
309 
310 ParseResult FCmpOp::parse(OpAsmParser &parser, OperationState &result) {
311  return parseCmpOp<FCmpPredicate>(parser, result);
312 }
313 
314 /// Returns a scalar or vector boolean attribute of the given type.
315 static Attribute getBoolAttribute(Type type, MLIRContext *ctx, bool value) {
316  auto boolAttr = BoolAttr::get(ctx, value);
317  ShapedType shapedType = dyn_cast<ShapedType>(type);
318  if (!shapedType)
319  return boolAttr;
320  return DenseElementsAttr::get(shapedType, boolAttr);
321 }
322 
323 OpFoldResult ICmpOp::fold(FoldAdaptor adaptor) {
324  if (getPredicate() != ICmpPredicate::eq &&
325  getPredicate() != ICmpPredicate::ne)
326  return {};
327 
328  // cmpi(eq/ne, x, x) -> true/false
329  if (getLhs() == getRhs())
330  return getBoolAttribute(getType(), getContext(),
331  getPredicate() == ICmpPredicate::eq);
332 
333  // cmpi(eq/ne, alloca, null) -> false/true
334  if (getLhs().getDefiningOp<AllocaOp>() && getRhs().getDefiningOp<ZeroOp>())
335  return getBoolAttribute(getType(), getContext(),
336  getPredicate() == ICmpPredicate::ne);
337 
338  // cmpi(eq/ne, null, alloca) -> cmpi(eq/ne, alloca, null)
339  if (getLhs().getDefiningOp<ZeroOp>() && getRhs().getDefiningOp<AllocaOp>()) {
340  Value lhs = getLhs();
341  Value rhs = getRhs();
342  getLhsMutable().assign(rhs);
343  getRhsMutable().assign(lhs);
344  return getResult();
345  }
346 
347  return {};
348 }
349 
350 //===----------------------------------------------------------------------===//
351 // Printing, parsing and verification for LLVM::AllocaOp.
352 //===----------------------------------------------------------------------===//
353 
354 void AllocaOp::print(OpAsmPrinter &p) {
355  auto funcTy =
356  FunctionType::get(getContext(), {getArraySize().getType()}, {getType()});
357 
358  if (getInalloca())
359  p << " inalloca";
360 
361  p << ' ' << getArraySize() << " x " << getElemType();
362  if (getAlignment() && *getAlignment() != 0)
363  p.printOptionalAttrDict((*this)->getAttrs(),
364  {kElemTypeAttrName, getInallocaAttrName()});
365  else
367  (*this)->getAttrs(),
368  {getAlignmentAttrName(), kElemTypeAttrName, getInallocaAttrName()});
369  p << " : " << funcTy;
370 }
371 
372 // <operation> ::= `llvm.alloca` `inalloca`? ssa-use `x` type
373 // attribute-dict? `:` type `,` type
374 ParseResult AllocaOp::parse(OpAsmParser &parser, OperationState &result) {
376  Type type, elemType;
377  SMLoc trailingTypeLoc;
378 
379  if (succeeded(parser.parseOptionalKeyword("inalloca")))
380  result.addAttribute(getInallocaAttrName(result.name),
381  UnitAttr::get(parser.getContext()));
382 
383  if (parser.parseOperand(arraySize) || parser.parseKeyword("x") ||
384  parser.parseType(elemType) ||
385  parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
386  parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type))
387  return failure();
388 
389  std::optional<NamedAttribute> alignmentAttr =
390  result.attributes.getNamed("alignment");
391  if (alignmentAttr.has_value()) {
392  auto alignmentInt = llvm::dyn_cast<IntegerAttr>(alignmentAttr->getValue());
393  if (!alignmentInt)
394  return parser.emitError(parser.getNameLoc(),
395  "expected integer alignment");
396  if (alignmentInt.getValue().isZero())
397  result.attributes.erase("alignment");
398  }
399 
400  // Extract the result type from the trailing function type.
401  auto funcType = llvm::dyn_cast<FunctionType>(type);
402  if (!funcType || funcType.getNumInputs() != 1 ||
403  funcType.getNumResults() != 1)
404  return parser.emitError(
405  trailingTypeLoc,
406  "expected trailing function type with one argument and one result");
407 
408  if (parser.resolveOperand(arraySize, funcType.getInput(0), result.operands))
409  return failure();
410 
411  Type resultType = funcType.getResult(0);
412  if (auto ptrResultType = llvm::dyn_cast<LLVMPointerType>(resultType))
413  result.addAttribute(kElemTypeAttrName, TypeAttr::get(elemType));
414 
415  result.addTypes({funcType.getResult(0)});
416  return success();
417 }
418 
419 LogicalResult AllocaOp::verify() {
420  // Only certain target extension types can be used in 'alloca'.
421  if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getElemType());
422  targetExtType && !targetExtType.supportsMemOps())
423  return emitOpError()
424  << "this target extension type cannot be used in alloca";
425 
426  return success();
427 }
428 
429 //===----------------------------------------------------------------------===//
430 // LLVM::BrOp
431 //===----------------------------------------------------------------------===//
432 
434  assert(index == 0 && "invalid successor index");
435  return SuccessorOperands(getDestOperandsMutable());
436 }
437 
438 //===----------------------------------------------------------------------===//
439 // LLVM::CondBrOp
440 //===----------------------------------------------------------------------===//
441 
443  assert(index < getNumSuccessors() && "invalid successor index");
444  return SuccessorOperands(index == 0 ? getTrueDestOperandsMutable()
445  : getFalseDestOperandsMutable());
446 }
447 
448 void CondBrOp::build(OpBuilder &builder, OperationState &result,
449  Value condition, Block *trueDest, ValueRange trueOperands,
450  Block *falseDest, ValueRange falseOperands,
451  std::optional<std::pair<uint32_t, uint32_t>> weights) {
452  DenseI32ArrayAttr weightsAttr;
453  if (weights)
454  weightsAttr =
455  builder.getDenseI32ArrayAttr({static_cast<int32_t>(weights->first),
456  static_cast<int32_t>(weights->second)});
457 
458  build(builder, result, condition, trueOperands, falseOperands, weightsAttr,
459  /*loop_annotation=*/{}, trueDest, falseDest);
460 }
461 
462 //===----------------------------------------------------------------------===//
463 // LLVM::SwitchOp
464 //===----------------------------------------------------------------------===//
465 
466 void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
467  Block *defaultDestination, ValueRange defaultOperands,
468  DenseIntElementsAttr caseValues,
469  BlockRange caseDestinations,
470  ArrayRef<ValueRange> caseOperands,
471  ArrayRef<int32_t> branchWeights) {
472  DenseI32ArrayAttr weightsAttr;
473  if (!branchWeights.empty())
474  weightsAttr = builder.getDenseI32ArrayAttr(branchWeights);
475 
476  build(builder, result, value, defaultOperands, caseOperands, caseValues,
477  weightsAttr, defaultDestination, caseDestinations);
478 }
479 
480 void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
481  Block *defaultDestination, ValueRange defaultOperands,
482  ArrayRef<APInt> caseValues, BlockRange caseDestinations,
483  ArrayRef<ValueRange> caseOperands,
484  ArrayRef<int32_t> branchWeights) {
485  DenseIntElementsAttr caseValuesAttr;
486  if (!caseValues.empty()) {
487  ShapedType caseValueType = VectorType::get(
488  static_cast<int64_t>(caseValues.size()), value.getType());
489  caseValuesAttr = DenseIntElementsAttr::get(caseValueType, caseValues);
490  }
491 
492  build(builder, result, value, defaultDestination, defaultOperands,
493  caseValuesAttr, caseDestinations, caseOperands, branchWeights);
494 }
495 
496 void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
497  Block *defaultDestination, ValueRange defaultOperands,
498  ArrayRef<int32_t> caseValues, BlockRange caseDestinations,
499  ArrayRef<ValueRange> caseOperands,
500  ArrayRef<int32_t> branchWeights) {
501  DenseIntElementsAttr caseValuesAttr;
502  if (!caseValues.empty()) {
503  ShapedType caseValueType = VectorType::get(
504  static_cast<int64_t>(caseValues.size()), value.getType());
505  caseValuesAttr = DenseIntElementsAttr::get(caseValueType, caseValues);
506  }
507 
508  build(builder, result, value, defaultDestination, defaultOperands,
509  caseValuesAttr, caseDestinations, caseOperands, branchWeights);
510 }
511 
512 /// <cases> ::= `[` (case (`,` case )* )? `]`
513 /// <case> ::= integer `:` bb-id (`(` ssa-use-and-type-list `)`)?
514 static ParseResult parseSwitchOpCases(
515  OpAsmParser &parser, Type flagType, DenseIntElementsAttr &caseValues,
516  SmallVectorImpl<Block *> &caseDestinations,
518  SmallVectorImpl<SmallVector<Type>> &caseOperandTypes) {
519  if (failed(parser.parseLSquare()))
520  return failure();
521  if (succeeded(parser.parseOptionalRSquare()))
522  return success();
523  SmallVector<APInt> values;
524  unsigned bitWidth = flagType.getIntOrFloatBitWidth();
525  auto parseCase = [&]() {
526  int64_t value = 0;
527  if (failed(parser.parseInteger(value)))
528  return failure();
529  values.push_back(APInt(bitWidth, value, /*isSigned=*/true));
530 
531  Block *destination;
533  SmallVector<Type> operandTypes;
534  if (parser.parseColon() || parser.parseSuccessor(destination))
535  return failure();
536  if (!parser.parseOptionalLParen()) {
537  if (parser.parseOperandList(operands, OpAsmParser::Delimiter::None,
538  /*allowResultNumber=*/false) ||
539  parser.parseColonTypeList(operandTypes) || parser.parseRParen())
540  return failure();
541  }
542  caseDestinations.push_back(destination);
543  caseOperands.emplace_back(operands);
544  caseOperandTypes.emplace_back(operandTypes);
545  return success();
546  };
547  if (failed(parser.parseCommaSeparatedList(parseCase)))
548  return failure();
549 
550  ShapedType caseValueType =
551  VectorType::get(static_cast<int64_t>(values.size()), flagType);
552  caseValues = DenseIntElementsAttr::get(caseValueType, values);
553  return parser.parseRSquare();
554 }
555 
556 static void printSwitchOpCases(OpAsmPrinter &p, SwitchOp op, Type flagType,
557  DenseIntElementsAttr caseValues,
558  SuccessorRange caseDestinations,
559  OperandRangeRange caseOperands,
560  const TypeRangeRange &caseOperandTypes) {
561  p << '[';
562  p.printNewline();
563  if (!caseValues) {
564  p << ']';
565  return;
566  }
567 
568  size_t index = 0;
569  llvm::interleave(
570  llvm::zip(caseValues, caseDestinations),
571  [&](auto i) {
572  p << " ";
573  p << std::get<0>(i);
574  p << ": ";
575  p.printSuccessorAndUseList(std::get<1>(i), caseOperands[index++]);
576  },
577  [&] {
578  p << ',';
579  p.printNewline();
580  });
581  p.printNewline();
582  p << ']';
583 }
584 
585 LogicalResult SwitchOp::verify() {
586  if ((!getCaseValues() && !getCaseDestinations().empty()) ||
587  (getCaseValues() &&
588  getCaseValues()->size() !=
589  static_cast<int64_t>(getCaseDestinations().size())))
590  return emitOpError("expects number of case values to match number of "
591  "case destinations");
592  if (getBranchWeights() && getBranchWeights()->size() != getNumSuccessors())
593  return emitError("expects number of branch weights to match number of "
594  "successors: ")
595  << getBranchWeights()->size() << " vs " << getNumSuccessors();
596  if (getCaseValues() &&
597  getValue().getType() != getCaseValues()->getElementType())
598  return emitError("expects case value type to match condition value type");
599  return success();
600 }
601 
603  assert(index < getNumSuccessors() && "invalid successor index");
604  return SuccessorOperands(index == 0 ? getDefaultOperandsMutable()
605  : getCaseOperandsMutable(index - 1));
606 }
607 
608 //===----------------------------------------------------------------------===//
609 // Code for LLVM::GEPOp.
610 //===----------------------------------------------------------------------===//
611 
612 constexpr int32_t GEPOp::kDynamicIndex;
613 
615  return GEPIndicesAdaptor<ValueRange>(getRawConstantIndicesAttr(),
616  getDynamicIndices());
617 }
618 
619 /// Returns the elemental type of any LLVM-compatible vector type or self.
621  if (auto vectorType = llvm::dyn_cast<VectorType>(type))
622  return vectorType.getElementType();
623  return type;
624 }
625 
626 /// Destructures the 'indices' parameter into 'rawConstantIndices' and
627 /// 'dynamicIndices', encoding the former in the process. In the process,
628 /// dynamic indices which are used to index into a structure type are converted
629 /// to constant indices when possible. To do this, the GEPs element type should
630 /// be passed as first parameter.
631 static void destructureIndices(Type currType, ArrayRef<GEPArg> indices,
632  SmallVectorImpl<int32_t> &rawConstantIndices,
633  SmallVectorImpl<Value> &dynamicIndices) {
634  for (const GEPArg &iter : indices) {
635  // If the thing we are currently indexing into is a struct we must turn
636  // any integer constants into constant indices. If this is not possible
637  // we don't do anything here. The verifier will catch it and emit a proper
638  // error. All other canonicalization is done in the fold method.
639  bool requiresConst = !rawConstantIndices.empty() &&
640  isa_and_nonnull<LLVMStructType>(currType);
641  if (Value val = llvm::dyn_cast_if_present<Value>(iter)) {
642  APInt intC;
643  if (requiresConst && matchPattern(val, m_ConstantInt(&intC)) &&
644  intC.isSignedIntN(kGEPConstantBitWidth)) {
645  rawConstantIndices.push_back(intC.getSExtValue());
646  } else {
647  rawConstantIndices.push_back(GEPOp::kDynamicIndex);
648  dynamicIndices.push_back(val);
649  }
650  } else {
651  rawConstantIndices.push_back(cast<GEPConstantIndex>(iter));
652  }
653 
654  // Skip for very first iteration of this loop. First index does not index
655  // within the aggregates, but is just a pointer offset.
656  if (rawConstantIndices.size() == 1 || !currType)
657  continue;
658 
659  currType = TypeSwitch<Type, Type>(currType)
660  .Case<VectorType, LLVMArrayType>([](auto containerType) {
661  return containerType.getElementType();
662  })
663  .Case([&](LLVMStructType structType) -> Type {
664  int64_t memberIndex = rawConstantIndices.back();
665  if (memberIndex >= 0 && static_cast<size_t>(memberIndex) <
666  structType.getBody().size())
667  return structType.getBody()[memberIndex];
668  return nullptr;
669  })
670  .Default(Type(nullptr));
671  }
672 }
673 
674 void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
675  Type elementType, Value basePtr, ArrayRef<GEPArg> indices,
676  bool inbounds, ArrayRef<NamedAttribute> attributes) {
677  SmallVector<int32_t> rawConstantIndices;
678  SmallVector<Value> dynamicIndices;
679  destructureIndices(elementType, indices, rawConstantIndices, dynamicIndices);
680 
681  result.addTypes(resultType);
682  result.addAttributes(attributes);
683  result.addAttribute(getRawConstantIndicesAttrName(result.name),
684  builder.getDenseI32ArrayAttr(rawConstantIndices));
685  if (inbounds) {
686  result.addAttribute(getInboundsAttrName(result.name),
687  builder.getUnitAttr());
688  }
689  result.addAttribute(kElemTypeAttrName, TypeAttr::get(elementType));
690  result.addOperands(basePtr);
691  result.addOperands(dynamicIndices);
692 }
693 
694 void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
695  Type elementType, Value basePtr, ValueRange indices,
696  bool inbounds, ArrayRef<NamedAttribute> attributes) {
697  build(builder, result, resultType, elementType, basePtr,
698  SmallVector<GEPArg>(indices), inbounds, attributes);
699 }
700 
701 static ParseResult
704  DenseI32ArrayAttr &rawConstantIndices) {
705  SmallVector<int32_t> constantIndices;
706 
707  auto idxParser = [&]() -> ParseResult {
708  int32_t constantIndex;
709  OptionalParseResult parsedInteger =
711  if (parsedInteger.has_value()) {
712  if (failed(parsedInteger.value()))
713  return failure();
714  constantIndices.push_back(constantIndex);
715  return success();
716  }
717 
718  constantIndices.push_back(LLVM::GEPOp::kDynamicIndex);
719  return parser.parseOperand(indices.emplace_back());
720  };
721  if (parser.parseCommaSeparatedList(idxParser))
722  return failure();
723 
724  rawConstantIndices =
725  DenseI32ArrayAttr::get(parser.getContext(), constantIndices);
726  return success();
727 }
728 
729 static void printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp,
730  OperandRange indices,
731  DenseI32ArrayAttr rawConstantIndices) {
732  llvm::interleaveComma(
733  GEPIndicesAdaptor<OperandRange>(rawConstantIndices, indices), printer,
735  if (Value val = llvm::dyn_cast_if_present<Value>(cst))
736  printer.printOperand(val);
737  else
738  printer << cast<IntegerAttr>(cst).getInt();
739  });
740 }
741 
742 /// For the given `indices`, check if they comply with `baseGEPType`,
743 /// especially check against LLVMStructTypes nested within.
744 static LogicalResult
745 verifyStructIndices(Type baseGEPType, unsigned indexPos,
747  function_ref<InFlightDiagnostic()> emitOpError) {
748  if (indexPos >= indices.size())
749  // Stop searching
750  return success();
751 
752  return TypeSwitch<Type, LogicalResult>(baseGEPType)
753  .Case<LLVMStructType>([&](LLVMStructType structType) -> LogicalResult {
754  auto attr = dyn_cast<IntegerAttr>(indices[indexPos]);
755  if (!attr)
756  return emitOpError() << "expected index " << indexPos
757  << " indexing a struct to be constant";
758 
759  int32_t gepIndex = attr.getInt();
760  ArrayRef<Type> elementTypes = structType.getBody();
761  if (gepIndex < 0 ||
762  static_cast<size_t>(gepIndex) >= elementTypes.size())
763  return emitOpError() << "index " << indexPos
764  << " indexing a struct is out of bounds";
765 
766  // Instead of recursively going into every children types, we only
767  // dive into the one indexed by gepIndex.
768  return verifyStructIndices(elementTypes[gepIndex], indexPos + 1,
769  indices, emitOpError);
770  })
771  .Case<VectorType, LLVMArrayType>(
772  [&](auto containerType) -> LogicalResult {
773  return verifyStructIndices(containerType.getElementType(),
774  indexPos + 1, indices, emitOpError);
775  })
776  .Default([&](auto otherType) -> LogicalResult {
777  return emitOpError()
778  << "type " << otherType << " cannot be indexed (index #"
779  << indexPos << ")";
780  });
781 }
782 
783 /// Driver function around `verifyStructIndices`.
784 static LogicalResult
786  function_ref<InFlightDiagnostic()> emitOpError) {
787  return verifyStructIndices(baseGEPType, /*indexPos=*/1, indices, emitOpError);
788 }
789 
790 LogicalResult LLVM::GEPOp::verify() {
791  if (static_cast<size_t>(
792  llvm::count(getRawConstantIndices(), kDynamicIndex)) !=
793  getDynamicIndices().size())
794  return emitOpError("expected as many dynamic indices as specified in '")
795  << getRawConstantIndicesAttrName().getValue() << "'";
796 
797  return verifyStructIndices(getElemType(), getIndices(),
798  [&] { return emitOpError(); });
799 }
800 
801 //===----------------------------------------------------------------------===//
802 // LoadOp
803 //===----------------------------------------------------------------------===//
804 
805 void LoadOp::getEffects(
807  &effects) {
808  effects.emplace_back(MemoryEffects::Read::get(), &getAddrMutable());
809  // Volatile operations can have target-specific read-write effects on
810  // memory besides the one referred to by the pointer operand.
811  // Similarly, atomic operations that are monotonic or stricter cause
812  // synchronization that from a language point-of-view, are arbitrary
813  // read-writes into memory.
814  if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
815  getOrdering() != AtomicOrdering::unordered)) {
816  effects.emplace_back(MemoryEffects::Write::get());
817  effects.emplace_back(MemoryEffects::Read::get());
818  }
819 }
820 
821 /// Returns true if the given type is supported by atomic operations. All
822 /// integer, float, and pointer types with a power-of-two bitsize and a minimal
823 /// size of 8 bits are supported.
825  const DataLayout &dataLayout) {
826  if (!isa<IntegerType, LLVMPointerType>(type))
828  return false;
829 
830  llvm::TypeSize bitWidth = dataLayout.getTypeSizeInBits(type);
831  if (bitWidth.isScalable())
832  return false;
833  // Needs to be at least 8 bits and a power of two.
834  return bitWidth >= 8 && (bitWidth & (bitWidth - 1)) == 0;
835 }
836 
837 /// Verifies the attributes and the type of atomic memory access operations.
838 template <typename OpTy>
839 LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType,
840  ArrayRef<AtomicOrdering> unsupportedOrderings) {
841  if (memOp.getOrdering() != AtomicOrdering::not_atomic) {
842  DataLayout dataLayout = DataLayout::closest(memOp);
843  if (!isTypeCompatibleWithAtomicOp(valueType, dataLayout))
844  return memOp.emitOpError("unsupported type ")
845  << valueType << " for atomic access";
846  if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering()))
847  return memOp.emitOpError("unsupported ordering '")
848  << stringifyAtomicOrdering(memOp.getOrdering()) << "'";
849  if (!memOp.getAlignment())
850  return memOp.emitOpError("expected alignment for atomic access");
851  return success();
852  }
853  if (memOp.getSyncscope())
854  return memOp.emitOpError(
855  "expected syncscope to be null for non-atomic access");
856  return success();
857 }
858 
859 LogicalResult LoadOp::verify() {
860  Type valueType = getResult().getType();
861  return verifyAtomicMemOp(*this, valueType,
862  {AtomicOrdering::release, AtomicOrdering::acq_rel});
863 }
864 
865 void LoadOp::build(OpBuilder &builder, OperationState &state, Type type,
866  Value addr, unsigned alignment, bool isVolatile,
867  bool isNonTemporal, bool isInvariant, bool isInvariantGroup,
868  AtomicOrdering ordering, StringRef syncscope) {
869  build(builder, state, type, addr,
870  alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
871  isNonTemporal, isInvariant, isInvariantGroup, ordering,
872  syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
873  /*dereferenceable=*/nullptr,
874  /*access_groups=*/nullptr,
875  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr,
876  /*tbaa=*/nullptr);
877 }
878 
879 //===----------------------------------------------------------------------===//
880 // StoreOp
881 //===----------------------------------------------------------------------===//
882 
883 void StoreOp::getEffects(
885  &effects) {
886  effects.emplace_back(MemoryEffects::Write::get(), &getAddrMutable());
887  // Volatile operations can have target-specific read-write effects on
888  // memory besides the one referred to by the pointer operand.
889  // Similarly, atomic operations that are monotonic or stricter cause
890  // synchronization that from a language point-of-view, are arbitrary
891  // read-writes into memory.
892  if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
893  getOrdering() != AtomicOrdering::unordered)) {
894  effects.emplace_back(MemoryEffects::Write::get());
895  effects.emplace_back(MemoryEffects::Read::get());
896  }
897 }
898 
899 LogicalResult StoreOp::verify() {
900  Type valueType = getValue().getType();
901  return verifyAtomicMemOp(*this, valueType,
902  {AtomicOrdering::acquire, AtomicOrdering::acq_rel});
903 }
904 
905 void StoreOp::build(OpBuilder &builder, OperationState &state, Value value,
906  Value addr, unsigned alignment, bool isVolatile,
907  bool isNonTemporal, bool isInvariantGroup,
908  AtomicOrdering ordering, StringRef syncscope) {
909  build(builder, state, value, addr,
910  alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
911  isNonTemporal, isInvariantGroup, ordering,
912  syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
913  /*access_groups=*/nullptr,
914  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
915 }
916 
917 //===----------------------------------------------------------------------===//
918 // CallOp
919 //===----------------------------------------------------------------------===//
920 
921 /// Gets the MLIR Op-like result types of a LLVMFunctionType.
922 static SmallVector<Type, 1> getCallOpResultTypes(LLVMFunctionType calleeType) {
923  SmallVector<Type, 1> results;
924  Type resultType = calleeType.getReturnType();
925  if (!isa<LLVM::LLVMVoidType>(resultType))
926  results.push_back(resultType);
927  return results;
928 }
929 
930 /// Gets the variadic callee type for a LLVMFunctionType.
931 static TypeAttr getCallOpVarCalleeType(LLVMFunctionType calleeType) {
932  return calleeType.isVarArg() ? TypeAttr::get(calleeType) : nullptr;
933 }
934 
935 /// Constructs a LLVMFunctionType from MLIR `results` and `args`.
936 static LLVMFunctionType getLLVMFuncType(MLIRContext *context, TypeRange results,
937  ValueRange args) {
938  Type resultType;
939  if (results.empty())
940  resultType = LLVMVoidType::get(context);
941  else
942  resultType = results.front();
943  return LLVMFunctionType::get(resultType, llvm::to_vector(args.getTypes()),
944  /*isVarArg=*/false);
945 }
946 
947 void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
948  StringRef callee, ValueRange args) {
949  build(builder, state, results, builder.getStringAttr(callee), args);
950 }
951 
952 void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
953  StringAttr callee, ValueRange args) {
954  build(builder, state, results, SymbolRefAttr::get(callee), args);
955 }
956 
957 void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
958  FlatSymbolRefAttr callee, ValueRange args) {
959  assert(callee && "expected non-null callee in direct call builder");
960  build(builder, state, results,
961  /*var_callee_type=*/nullptr, callee, args, /*fastmathFlags=*/nullptr,
962  /*branch_weights=*/nullptr,
963  /*CConv=*/nullptr, /*TailCallKind=*/nullptr,
964  /*memory_effects=*/nullptr,
965  /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
966  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
967  /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
968  /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
969  /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
970  /*no_inline=*/nullptr, /*always_inline=*/nullptr,
971  /*inline_hint=*/nullptr);
972 }
973 
974 void CallOp::build(OpBuilder &builder, OperationState &state,
975  LLVMFunctionType calleeType, StringRef callee,
976  ValueRange args) {
977  build(builder, state, calleeType, builder.getStringAttr(callee), args);
978 }
979 
980 void CallOp::build(OpBuilder &builder, OperationState &state,
981  LLVMFunctionType calleeType, StringAttr callee,
982  ValueRange args) {
983  build(builder, state, calleeType, SymbolRefAttr::get(callee), args);
984 }
985 
986 void CallOp::build(OpBuilder &builder, OperationState &state,
987  LLVMFunctionType calleeType, FlatSymbolRefAttr callee,
988  ValueRange args) {
989  build(builder, state, getCallOpResultTypes(calleeType),
990  getCallOpVarCalleeType(calleeType), callee, args,
991  /*fastmathFlags=*/nullptr,
992  /*branch_weights=*/nullptr, /*CConv=*/nullptr,
993  /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
994  /*convergent=*/nullptr,
995  /*no_unwind=*/nullptr, /*will_return=*/nullptr,
996  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
997  /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
998  /*access_groups=*/nullptr,
999  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
1000  /*no_inline=*/nullptr, /*always_inline=*/nullptr,
1001  /*inline_hint=*/nullptr);
1002 }
1003 
1004 void CallOp::build(OpBuilder &builder, OperationState &state,
1005  LLVMFunctionType calleeType, ValueRange args) {
1006  build(builder, state, getCallOpResultTypes(calleeType),
1007  getCallOpVarCalleeType(calleeType),
1008  /*callee=*/nullptr, args,
1009  /*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
1010  /*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
1011  /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
1012  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1013  /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
1014  /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
1015  /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
1016  /*no_inline=*/nullptr, /*always_inline=*/nullptr,
1017  /*inline_hint=*/nullptr);
1018 }
1019 
1020 void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
1021  ValueRange args) {
1022  auto calleeType = func.getFunctionType();
1023  build(builder, state, getCallOpResultTypes(calleeType),
1024  getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), args,
1025  /*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
1026  /*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
1027  /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
1028  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1029  /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
1030  /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
1031  /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
1032  /*no_inline=*/nullptr, /*always_inline=*/nullptr,
1033  /*inline_hint=*/nullptr);
1034 }
1035 
1036 CallInterfaceCallable CallOp::getCallableForCallee() {
1037  // Direct call.
1038  if (FlatSymbolRefAttr calleeAttr = getCalleeAttr())
1039  return calleeAttr;
1040  // Indirect call, callee Value is the first operand.
1041  return getOperand(0);
1042 }
1043 
1044 void CallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
1045  // Direct call.
1046  if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) {
1047  auto symRef = cast<SymbolRefAttr>(callee);
1048  return setCalleeAttr(cast<FlatSymbolRefAttr>(symRef));
1049  }
1050  // Indirect call, callee Value is the first operand.
1051  return setOperand(0, cast<Value>(callee));
1052 }
1053 
1054 Operation::operand_range CallOp::getArgOperands() {
1055  return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
1056 }
1057 
1058 MutableOperandRange CallOp::getArgOperandsMutable() {
1059  return MutableOperandRange(*this, getCallee().has_value() ? 0 : 1,
1060  getCalleeOperands().size());
1061 }
1062 
1063 /// Verify that an inlinable callsite of a debug-info-bearing function in a
1064 /// debug-info-bearing function has a debug location attached to it. This
1065 /// mirrors an LLVM IR verifier.
1066 static LogicalResult verifyCallOpDebugInfo(CallOp callOp, LLVMFuncOp callee) {
1067  if (callee.isExternal())
1068  return success();
1069  auto parentFunc = callOp->getParentOfType<FunctionOpInterface>();
1070  if (!parentFunc)
1071  return success();
1072 
1073  auto hasSubprogram = [](Operation *op) {
1074  return op->getLoc()
1075  ->findInstanceOf<FusedLocWith<LLVM::DISubprogramAttr>>() !=
1076  nullptr;
1077  };
1078  if (!hasSubprogram(parentFunc) || !hasSubprogram(callee))
1079  return success();
1080  bool containsLoc = !isa<UnknownLoc>(callOp->getLoc());
1081  if (!containsLoc)
1082  return callOp.emitError()
1083  << "inlinable function call in a function with a DISubprogram "
1084  "location must have a debug location";
1085  return success();
1086 }
1087 
1088 /// Verify that the parameter and return types of the variadic callee type match
1089 /// the `callOp` argument and result types.
1090 template <typename OpTy>
1091 LogicalResult verifyCallOpVarCalleeType(OpTy callOp) {
1092  std::optional<LLVMFunctionType> varCalleeType = callOp.getVarCalleeType();
1093  if (!varCalleeType)
1094  return success();
1095 
1096  // Verify the variadic callee type is a variadic function type.
1097  if (!varCalleeType->isVarArg())
1098  return callOp.emitOpError(
1099  "expected var_callee_type to be a variadic function type");
1100 
1101  // Verify the variadic callee type has at most as many parameters as the call
1102  // has argument operands.
1103  if (varCalleeType->getNumParams() > callOp.getArgOperands().size())
1104  return callOp.emitOpError("expected var_callee_type to have at most ")
1105  << callOp.getArgOperands().size() << " parameters";
1106 
1107  // Verify the variadic callee type matches the call argument types.
1108  for (auto [paramType, operand] :
1109  llvm::zip(varCalleeType->getParams(), callOp.getArgOperands()))
1110  if (paramType != operand.getType())
1111  return callOp.emitOpError()
1112  << "var_callee_type parameter type mismatch: " << paramType
1113  << " != " << operand.getType();
1114 
1115  // Verify the variadic callee type matches the call result type.
1116  if (!callOp.getNumResults()) {
1117  if (!isa<LLVMVoidType>(varCalleeType->getReturnType()))
1118  return callOp.emitOpError("expected var_callee_type to return void");
1119  } else {
1120  if (callOp.getResult().getType() != varCalleeType->getReturnType())
1121  return callOp.emitOpError("var_callee_type return type mismatch: ")
1122  << varCalleeType->getReturnType()
1123  << " != " << callOp.getResult().getType();
1124  }
1125  return success();
1126 }
1127 
1128 template <typename OpType>
1129 static LogicalResult verifyOperandBundles(OpType &op) {
1130  OperandRangeRange opBundleOperands = op.getOpBundleOperands();
1131  std::optional<ArrayAttr> opBundleTags = op.getOpBundleTags();
1132 
1133  auto isStringAttr = [](Attribute tagAttr) {
1134  return isa<StringAttr>(tagAttr);
1135  };
1136  if (opBundleTags && !llvm::all_of(*opBundleTags, isStringAttr))
1137  return op.emitError("operand bundle tag must be a StringAttr");
1138 
1139  size_t numOpBundles = opBundleOperands.size();
1140  size_t numOpBundleTags = opBundleTags ? opBundleTags->size() : 0;
1141  if (numOpBundles != numOpBundleTags)
1142  return op.emitError("expected ")
1143  << numOpBundles << " operand bundle tags, but actually got "
1144  << numOpBundleTags;
1145 
1146  return success();
1147 }
1148 
1149 LogicalResult CallOp::verify() { return verifyOperandBundles(*this); }
1150 
1151 LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1152  if (failed(verifyCallOpVarCalleeType(*this)))
1153  return failure();
1154 
1155  // Type for the callee, we'll get it differently depending if it is a direct
1156  // or indirect call.
1157  Type fnType;
1158 
1159  bool isIndirect = false;
1160 
1161  // If this is an indirect call, the callee attribute is missing.
1162  FlatSymbolRefAttr calleeName = getCalleeAttr();
1163  if (!calleeName) {
1164  isIndirect = true;
1165  if (!getNumOperands())
1166  return emitOpError(
1167  "must have either a `callee` attribute or at least an operand");
1168  auto ptrType = llvm::dyn_cast<LLVMPointerType>(getOperand(0).getType());
1169  if (!ptrType)
1170  return emitOpError("indirect call expects a pointer as callee: ")
1171  << getOperand(0).getType();
1172 
1173  return success();
1174  } else {
1175  Operation *callee =
1176  symbolTable.lookupNearestSymbolFrom(*this, calleeName.getAttr());
1177  if (!callee)
1178  return emitOpError()
1179  << "'" << calleeName.getValue()
1180  << "' does not reference a symbol in the current scope";
1181  auto fn = dyn_cast<LLVMFuncOp>(callee);
1182  if (!fn)
1183  return emitOpError() << "'" << calleeName.getValue()
1184  << "' does not reference a valid LLVM function";
1185 
1186  if (failed(verifyCallOpDebugInfo(*this, fn)))
1187  return failure();
1188  fnType = fn.getFunctionType();
1189  }
1190 
1191  LLVMFunctionType funcType = llvm::dyn_cast<LLVMFunctionType>(fnType);
1192  if (!funcType)
1193  return emitOpError("callee does not have a functional type: ") << fnType;
1194 
1195  if (funcType.isVarArg() && !getVarCalleeType())
1196  return emitOpError() << "missing var_callee_type attribute for vararg call";
1197 
1198  // Verify that the operand and result types match the callee.
1199 
1200  if (!funcType.isVarArg() &&
1201  funcType.getNumParams() != (getCalleeOperands().size() - isIndirect))
1202  return emitOpError() << "incorrect number of operands ("
1203  << (getCalleeOperands().size() - isIndirect)
1204  << ") for callee (expecting: "
1205  << funcType.getNumParams() << ")";
1206 
1207  if (funcType.getNumParams() > (getCalleeOperands().size() - isIndirect))
1208  return emitOpError() << "incorrect number of operands ("
1209  << (getCalleeOperands().size() - isIndirect)
1210  << ") for varargs callee (expecting at least: "
1211  << funcType.getNumParams() << ")";
1212 
1213  for (unsigned i = 0, e = funcType.getNumParams(); i != e; ++i)
1214  if (getOperand(i + isIndirect).getType() != funcType.getParamType(i))
1215  return emitOpError() << "operand type mismatch for operand " << i << ": "
1216  << getOperand(i + isIndirect).getType()
1217  << " != " << funcType.getParamType(i);
1218 
1219  if (getNumResults() == 0 &&
1220  !llvm::isa<LLVM::LLVMVoidType>(funcType.getReturnType()))
1221  return emitOpError() << "expected function call to produce a value";
1222 
1223  if (getNumResults() != 0 &&
1224  llvm::isa<LLVM::LLVMVoidType>(funcType.getReturnType()))
1225  return emitOpError()
1226  << "calling function with void result must not produce values";
1227 
1228  if (getNumResults() > 1)
1229  return emitOpError()
1230  << "expected LLVM function call to produce 0 or 1 result";
1231 
1232  if (getNumResults() && getResult().getType() != funcType.getReturnType())
1233  return emitOpError() << "result type mismatch: " << getResult().getType()
1234  << " != " << funcType.getReturnType();
1235 
1236  return success();
1237 }
1238 
1239 void CallOp::print(OpAsmPrinter &p) {
1240  auto callee = getCallee();
1241  bool isDirect = callee.has_value();
1242 
1243  p << ' ';
1244 
1245  // Print calling convention.
1246  if (getCConv() != LLVM::CConv::C)
1247  p << stringifyCConv(getCConv()) << ' ';
1248 
1249  if (getTailCallKind() != LLVM::TailCallKind::None)
1250  p << tailcallkind::stringifyTailCallKind(getTailCallKind()) << ' ';
1251 
1252  // Print the direct callee if present as a function attribute, or an indirect
1253  // callee (first operand) otherwise.
1254  if (isDirect)
1255  p.printSymbolName(callee.value());
1256  else
1257  p << getOperand(0);
1258 
1259  auto args = getCalleeOperands().drop_front(isDirect ? 0 : 1);
1260  p << '(' << args << ')';
1261 
1262  // Print the variadic callee type if the call is variadic.
1263  if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1264  p << " vararg(" << *varCalleeType << ")";
1265 
1266  if (!getOpBundleOperands().empty()) {
1267  p << " ";
1268  printOpBundles(p, *this, getOpBundleOperands(),
1269  getOpBundleOperands().getTypes(), getOpBundleTags());
1270  }
1271 
1272  p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
1273  {getCalleeAttrName(), getTailCallKindAttrName(),
1274  getVarCalleeTypeAttrName(), getCConvAttrName(),
1275  getOperandSegmentSizesAttrName(),
1276  getOpBundleSizesAttrName(),
1277  getOpBundleTagsAttrName(), getArgAttrsAttrName(),
1278  getResAttrsAttrName()});
1279 
1280  p << " : ";
1281  if (!isDirect)
1282  p << getOperand(0).getType() << ", ";
1283 
1284  // Reconstruct the MLIR function type from operand and result types.
1286  p, args.getTypes(), getArgAttrsAttr(),
1287  /*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
1288 }
1289 
1290 /// Parses the type of a call operation and resolves the operands if the parsing
1291 /// succeeds. Returns failure otherwise.
1293  OpAsmParser &parser, OperationState &result, bool isDirect,
1296  SmallVectorImpl<DictionaryAttr> &resultAttrs) {
1297  SMLoc trailingTypesLoc = parser.getCurrentLocation();
1298  SmallVector<Type> types;
1299  if (parser.parseColon())
1300  return failure();
1301  if (!isDirect) {
1302  types.emplace_back();
1303  if (parser.parseType(types.back()))
1304  return failure();
1305  if (parser.parseOptionalComma())
1306  return parser.emitError(
1307  trailingTypesLoc, "expected indirect call to have 2 trailing types");
1308  }
1309  SmallVector<Type> argTypes;
1310  SmallVector<Type> resTypes;
1311  if (call_interface_impl::parseFunctionSignature(parser, argTypes, argAttrs,
1312  resTypes, resultAttrs)) {
1313  if (isDirect)
1314  return parser.emitError(trailingTypesLoc,
1315  "expected direct call to have 1 trailing types");
1316  return parser.emitError(trailingTypesLoc,
1317  "expected trailing function type");
1318  }
1319 
1320  if (resTypes.size() > 1)
1321  return parser.emitError(trailingTypesLoc,
1322  "expected function with 0 or 1 result");
1323  if (resTypes.size() == 1 && llvm::isa<LLVM::LLVMVoidType>(resTypes[0]))
1324  return parser.emitError(trailingTypesLoc,
1325  "expected a non-void result type");
1326 
1327  // The head element of the types list matches the callee type for
1328  // indirect calls, while the types list is emtpy for direct calls.
1329  // Append the function input types to resolve the call operation
1330  // operands.
1331  llvm::append_range(types, argTypes);
1332  if (parser.resolveOperands(operands, types, parser.getNameLoc(),
1333  result.operands))
1334  return failure();
1335  if (resTypes.size() != 0)
1336  result.addTypes(resTypes);
1337 
1338  return success();
1339 }
1340 
1341 /// Parses an optional function pointer operand before the call argument list
1342 /// for indirect calls, or stops parsing at the function identifier otherwise.
1343 static ParseResult parseOptionalCallFuncPtr(
1344  OpAsmParser &parser,
1346  OpAsmParser::UnresolvedOperand funcPtrOperand;
1347  OptionalParseResult parseResult = parser.parseOptionalOperand(funcPtrOperand);
1348  if (parseResult.has_value()) {
1349  if (failed(*parseResult))
1350  return *parseResult;
1351  operands.push_back(funcPtrOperand);
1352  }
1353  return success();
1354 }
1355 
1356 static ParseResult resolveOpBundleOperands(
1357  OpAsmParser &parser, SMLoc loc, OperationState &state,
1359  ArrayRef<SmallVector<Type>> opBundleOperandTypes,
1360  StringAttr opBundleSizesAttrName) {
1361  unsigned opBundleIndex = 0;
1362  for (const auto &[operands, types] :
1363  llvm::zip_equal(opBundleOperands, opBundleOperandTypes)) {
1364  if (operands.size() != types.size())
1365  return parser.emitError(loc, "expected ")
1366  << operands.size()
1367  << " types for operand bundle operands for operand bundle #"
1368  << opBundleIndex << ", but actually got " << types.size();
1369  if (parser.resolveOperands(operands, types, loc, state.operands))
1370  return failure();
1371  }
1372 
1373  SmallVector<int32_t> opBundleSizes;
1374  opBundleSizes.reserve(opBundleOperands.size());
1375  for (const auto &operands : opBundleOperands)
1376  opBundleSizes.push_back(operands.size());
1377 
1378  state.addAttribute(
1379  opBundleSizesAttrName,
1380  DenseI32ArrayAttr::get(parser.getContext(), opBundleSizes));
1381 
1382  return success();
1383 }
1384 
1385 // <operation> ::= `llvm.call` (cconv)? (tailcallkind)? (function-id | ssa-use)
1386 // `(` ssa-use-list `)`
1387 // ( `vararg(` var-callee-type `)` )?
1388 // ( `[` op-bundles-list `]` )?
1389 // attribute-dict? `:` (type `,`)? function-type
1390 ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
1391  SymbolRefAttr funcAttr;
1392  TypeAttr varCalleeType;
1395  SmallVector<SmallVector<Type>> opBundleOperandTypes;
1396  ArrayAttr opBundleTags;
1397 
1398  // Default to C Calling Convention if no keyword is provided.
1399  result.addAttribute(
1400  getCConvAttrName(result.name),
1401  CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
1402  parser, result, LLVM::CConv::C)));
1403 
1404  result.addAttribute(
1405  getTailCallKindAttrName(result.name),
1407  parseOptionalLLVMKeyword<TailCallKind>(
1408  parser, result, LLVM::TailCallKind::None)));
1409 
1410  // Parse a function pointer for indirect calls.
1411  if (parseOptionalCallFuncPtr(parser, operands))
1412  return failure();
1413  bool isDirect = operands.empty();
1414 
1415  // Parse a function identifier for direct calls.
1416  if (isDirect)
1417  if (parser.parseAttribute(funcAttr, "callee", result.attributes))
1418  return failure();
1419 
1420  // Parse the function arguments.
1421  if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren))
1422  return failure();
1423 
1424  bool isVarArg = parser.parseOptionalKeyword("vararg").succeeded();
1425  if (isVarArg) {
1426  StringAttr varCalleeTypeAttrName =
1427  CallOp::getVarCalleeTypeAttrName(result.name);
1428  if (parser.parseLParen().failed() ||
1429  parser
1430  .parseAttribute(varCalleeType, varCalleeTypeAttrName,
1431  result.attributes)
1432  .failed() ||
1433  parser.parseRParen().failed())
1434  return failure();
1435  }
1436 
1437  SMLoc opBundlesLoc = parser.getCurrentLocation();
1438  if (std::optional<ParseResult> result = parseOpBundles(
1439  parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
1440  result && failed(*result))
1441  return failure();
1442  if (opBundleTags && !opBundleTags.empty())
1443  result.addAttribute(CallOp::getOpBundleTagsAttrName(result.name).getValue(),
1444  opBundleTags);
1445 
1446  if (parser.parseOptionalAttrDict(result.attributes))
1447  return failure();
1448 
1449  // Parse the trailing type list and resolve the operands.
1450  SmallVector<DictionaryAttr> argAttrs;
1451  SmallVector<DictionaryAttr> resultAttrs;
1452  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands,
1453  argAttrs, resultAttrs))
1454  return failure();
1456  parser.getBuilder(), result, argAttrs, resultAttrs,
1457  getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
1458  if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
1459  opBundleOperandTypes,
1460  getOpBundleSizesAttrName(result.name)))
1461  return failure();
1462 
1463  int32_t numOpBundleOperands = 0;
1464  for (const auto &operands : opBundleOperands)
1465  numOpBundleOperands += operands.size();
1466 
1467  result.addAttribute(
1468  CallOp::getOperandSegmentSizeAttr(),
1470  {static_cast<int32_t>(operands.size()), numOpBundleOperands}));
1471  return success();
1472 }
1473 
1474 LLVMFunctionType CallOp::getCalleeFunctionType() {
1475  if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1476  return *varCalleeType;
1477  return getLLVMFuncType(getContext(), getResultTypes(), getArgOperands());
1478 }
1479 
1480 ///===---------------------------------------------------------------------===//
1481 /// LLVM::InvokeOp
1482 ///===---------------------------------------------------------------------===//
1483 
1484 void InvokeOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
1485  ValueRange ops, Block *normal, ValueRange normalOps,
1486  Block *unwind, ValueRange unwindOps) {
1487  auto calleeType = func.getFunctionType();
1488  build(builder, state, getCallOpResultTypes(calleeType),
1489  getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), ops,
1490  /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, normalOps, unwindOps,
1491  nullptr, nullptr, {}, {}, normal, unwind);
1492 }
1493 
1494 void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
1495  FlatSymbolRefAttr callee, ValueRange ops, Block *normal,
1496  ValueRange normalOps, Block *unwind,
1497  ValueRange unwindOps) {
1498  build(builder, state, tys,
1499  /*var_callee_type=*/nullptr, callee, ops, /*arg_attrs=*/nullptr,
1500  /*res_attrs=*/nullptr, normalOps, unwindOps, nullptr, nullptr, {}, {},
1501  normal, unwind);
1502 }
1503 
1504 void InvokeOp::build(OpBuilder &builder, OperationState &state,
1505  LLVMFunctionType calleeType, FlatSymbolRefAttr callee,
1506  ValueRange ops, Block *normal, ValueRange normalOps,
1507  Block *unwind, ValueRange unwindOps) {
1508  build(builder, state, getCallOpResultTypes(calleeType),
1509  getCallOpVarCalleeType(calleeType), callee, ops,
1510  /*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, normalOps, unwindOps,
1511  nullptr, nullptr, {}, {}, normal, unwind);
1512 }
1513 
1515  assert(index < getNumSuccessors() && "invalid successor index");
1516  return SuccessorOperands(index == 0 ? getNormalDestOperandsMutable()
1517  : getUnwindDestOperandsMutable());
1518 }
1519 
1520 CallInterfaceCallable InvokeOp::getCallableForCallee() {
1521  // Direct call.
1522  if (FlatSymbolRefAttr calleeAttr = getCalleeAttr())
1523  return calleeAttr;
1524  // Indirect call, callee Value is the first operand.
1525  return getOperand(0);
1526 }
1527 
1528 void InvokeOp::setCalleeFromCallable(CallInterfaceCallable callee) {
1529  // Direct call.
1530  if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) {
1531  auto symRef = cast<SymbolRefAttr>(callee);
1532  return setCalleeAttr(cast<FlatSymbolRefAttr>(symRef));
1533  }
1534  // Indirect call, callee Value is the first operand.
1535  return setOperand(0, cast<Value>(callee));
1536 }
1537 
1538 Operation::operand_range InvokeOp::getArgOperands() {
1539  return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
1540 }
1541 
1542 MutableOperandRange InvokeOp::getArgOperandsMutable() {
1543  return MutableOperandRange(*this, getCallee().has_value() ? 0 : 1,
1544  getCalleeOperands().size());
1545 }
1546 
1547 LogicalResult InvokeOp::verify() {
1548  if (failed(verifyCallOpVarCalleeType(*this)))
1549  return failure();
1550 
1551  Block *unwindDest = getUnwindDest();
1552  if (unwindDest->empty())
1553  return emitError("must have at least one operation in unwind destination");
1554 
1555  // In unwind destination, first operation must be LandingpadOp
1556  if (!isa<LandingpadOp>(unwindDest->front()))
1557  return emitError("first operation in unwind destination should be a "
1558  "llvm.landingpad operation");
1559 
1560  if (failed(verifyOperandBundles(*this)))
1561  return failure();
1562 
1563  return success();
1564 }
1565 
1566 void InvokeOp::print(OpAsmPrinter &p) {
1567  auto callee = getCallee();
1568  bool isDirect = callee.has_value();
1569 
1570  p << ' ';
1571 
1572  // Print calling convention.
1573  if (getCConv() != LLVM::CConv::C)
1574  p << stringifyCConv(getCConv()) << ' ';
1575 
1576  // Either function name or pointer
1577  if (isDirect)
1578  p.printSymbolName(callee.value());
1579  else
1580  p << getOperand(0);
1581 
1582  p << '(' << getCalleeOperands().drop_front(isDirect ? 0 : 1) << ')';
1583  p << " to ";
1584  p.printSuccessorAndUseList(getNormalDest(), getNormalDestOperands());
1585  p << " unwind ";
1586  p.printSuccessorAndUseList(getUnwindDest(), getUnwindDestOperands());
1587 
1588  // Print the variadic callee type if the invoke is variadic.
1589  if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1590  p << " vararg(" << *varCalleeType << ")";
1591 
1592  if (!getOpBundleOperands().empty()) {
1593  p << " ";
1594  printOpBundles(p, *this, getOpBundleOperands(),
1595  getOpBundleOperands().getTypes(), getOpBundleTags());
1596  }
1597 
1598  p.printOptionalAttrDict((*this)->getAttrs(),
1599  {getCalleeAttrName(), getOperandSegmentSizeAttr(),
1600  getCConvAttrName(), getVarCalleeTypeAttrName(),
1601  getOpBundleSizesAttrName(),
1602  getOpBundleTagsAttrName(), getArgAttrsAttrName(),
1603  getResAttrsAttrName()});
1604 
1605  p << " : ";
1606  if (!isDirect)
1607  p << getOperand(0).getType() << ", ";
1609  p, getCalleeOperands().drop_front(isDirect ? 0 : 1).getTypes(),
1610  getArgAttrsAttr(),
1611  /*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
1612 }
1613 
1614 // <operation> ::= `llvm.invoke` (cconv)? (function-id | ssa-use)
1615 // `(` ssa-use-list `)`
1616 // `to` bb-id (`[` ssa-use-and-type-list `]`)?
1617 // `unwind` bb-id (`[` ssa-use-and-type-list `]`)?
1618 // ( `vararg(` var-callee-type `)` )?
1619 // ( `[` op-bundles-list `]` )?
1620 // attribute-dict? `:` (type `,`)?
1621 // function-type-with-argument-attributes
1622 ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
1624  SymbolRefAttr funcAttr;
1625  TypeAttr varCalleeType;
1627  SmallVector<SmallVector<Type>> opBundleOperandTypes;
1628  ArrayAttr opBundleTags;
1629  Block *normalDest, *unwindDest;
1630  SmallVector<Value, 4> normalOperands, unwindOperands;
1631  Builder &builder = parser.getBuilder();
1632 
1633  // Default to C Calling Convention if no keyword is provided.
1634  result.addAttribute(
1635  getCConvAttrName(result.name),
1636  CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
1637  parser, result, LLVM::CConv::C)));
1638 
1639  // Parse a function pointer for indirect calls.
1640  if (parseOptionalCallFuncPtr(parser, operands))
1641  return failure();
1642  bool isDirect = operands.empty();
1643 
1644  // Parse a function identifier for direct calls.
1645  if (isDirect && parser.parseAttribute(funcAttr, "callee", result.attributes))
1646  return failure();
1647 
1648  // Parse the function arguments.
1649  if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) ||
1650  parser.parseKeyword("to") ||
1651  parser.parseSuccessorAndUseList(normalDest, normalOperands) ||
1652  parser.parseKeyword("unwind") ||
1653  parser.parseSuccessorAndUseList(unwindDest, unwindOperands))
1654  return failure();
1655 
1656  bool isVarArg = parser.parseOptionalKeyword("vararg").succeeded();
1657  if (isVarArg) {
1658  StringAttr varCalleeTypeAttrName =
1659  InvokeOp::getVarCalleeTypeAttrName(result.name);
1660  if (parser.parseLParen().failed() ||
1661  parser
1662  .parseAttribute(varCalleeType, varCalleeTypeAttrName,
1663  result.attributes)
1664  .failed() ||
1665  parser.parseRParen().failed())
1666  return failure();
1667  }
1668 
1669  SMLoc opBundlesLoc = parser.getCurrentLocation();
1670  if (std::optional<ParseResult> result = parseOpBundles(
1671  parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
1672  result && failed(*result))
1673  return failure();
1674  if (opBundleTags && !opBundleTags.empty())
1675  result.addAttribute(
1676  InvokeOp::getOpBundleTagsAttrName(result.name).getValue(),
1677  opBundleTags);
1678 
1679  if (parser.parseOptionalAttrDict(result.attributes))
1680  return failure();
1681 
1682  // Parse the trailing type list and resolve the function operands.
1683  SmallVector<DictionaryAttr> argAttrs;
1684  SmallVector<DictionaryAttr> resultAttrs;
1685  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands,
1686  argAttrs, resultAttrs))
1687  return failure();
1689  parser.getBuilder(), result, argAttrs, resultAttrs,
1690  getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
1691 
1692  if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
1693  opBundleOperandTypes,
1694  getOpBundleSizesAttrName(result.name)))
1695  return failure();
1696 
1697  result.addSuccessors({normalDest, unwindDest});
1698  result.addOperands(normalOperands);
1699  result.addOperands(unwindOperands);
1700 
1701  int32_t numOpBundleOperands = 0;
1702  for (const auto &operands : opBundleOperands)
1703  numOpBundleOperands += operands.size();
1704 
1705  result.addAttribute(
1706  InvokeOp::getOperandSegmentSizeAttr(),
1707  builder.getDenseI32ArrayAttr({static_cast<int32_t>(operands.size()),
1708  static_cast<int32_t>(normalOperands.size()),
1709  static_cast<int32_t>(unwindOperands.size()),
1710  numOpBundleOperands}));
1711  return success();
1712 }
1713 
1714 LLVMFunctionType InvokeOp::getCalleeFunctionType() {
1715  if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1716  return *varCalleeType;
1717  return getLLVMFuncType(getContext(), getResultTypes(), getArgOperands());
1718 }
1719 
1720 ///===----------------------------------------------------------------------===//
1721 /// Verifying/Printing/Parsing for LLVM::LandingpadOp.
1722 ///===----------------------------------------------------------------------===//
1723 
1724 LogicalResult LandingpadOp::verify() {
1725  Value value;
1726  if (LLVMFuncOp func = (*this)->getParentOfType<LLVMFuncOp>()) {
1727  if (!func.getPersonality())
1728  return emitError(
1729  "llvm.landingpad needs to be in a function with a personality");
1730  }
1731 
1732  // Consistency of llvm.landingpad result types is checked in
1733  // LLVMFuncOp::verify().
1734 
1735  if (!getCleanup() && getOperands().empty())
1736  return emitError("landingpad instruction expects at least one clause or "
1737  "cleanup attribute");
1738 
1739  for (unsigned idx = 0, ie = getNumOperands(); idx < ie; idx++) {
1740  value = getOperand(idx);
1741  bool isFilter = llvm::isa<LLVMArrayType>(value.getType());
1742  if (isFilter) {
1743  // FIXME: Verify filter clauses when arrays are appropriately handled
1744  } else {
1745  // catch - global addresses only.
1746  // Bitcast ops should have global addresses as their args.
1747  if (auto bcOp = value.getDefiningOp<BitcastOp>()) {
1748  if (auto addrOp = bcOp.getArg().getDefiningOp<AddressOfOp>())
1749  continue;
1750  return emitError("constant clauses expected").attachNote(bcOp.getLoc())
1751  << "global addresses expected as operand to "
1752  "bitcast used in clauses for landingpad";
1753  }
1754  // ZeroOp and AddressOfOp allowed
1755  if (value.getDefiningOp<ZeroOp>())
1756  continue;
1757  if (value.getDefiningOp<AddressOfOp>())
1758  continue;
1759  return emitError("clause #")
1760  << idx << " is not a known constant - null, addressof, bitcast";
1761  }
1762  }
1763  return success();
1764 }
1765 
1767  p << (getCleanup() ? " cleanup " : " ");
1768 
1769  // Clauses
1770  for (auto value : getOperands()) {
1771  // Similar to llvm - if clause is an array type then it is filter
1772  // clause else catch clause
1773  bool isArrayTy = llvm::isa<LLVMArrayType>(value.getType());
1774  p << '(' << (isArrayTy ? "filter " : "catch ") << value << " : "
1775  << value.getType() << ") ";
1776  }
1777 
1778  p.printOptionalAttrDict((*this)->getAttrs(), {"cleanup"});
1779 
1780  p << ": " << getType();
1781 }
1782 
1783 // <operation> ::= `llvm.landingpad` `cleanup`?
1784 // ((`catch` | `filter`) operand-type ssa-use)* attribute-dict?
1785 ParseResult LandingpadOp::parse(OpAsmParser &parser, OperationState &result) {
1786  // Check for cleanup
1787  if (succeeded(parser.parseOptionalKeyword("cleanup")))
1788  result.addAttribute("cleanup", parser.getBuilder().getUnitAttr());
1789 
1790  // Parse clauses with types
1791  while (succeeded(parser.parseOptionalLParen()) &&
1792  (succeeded(parser.parseOptionalKeyword("filter")) ||
1793  succeeded(parser.parseOptionalKeyword("catch")))) {
1795  Type ty;
1796  if (parser.parseOperand(operand) || parser.parseColon() ||
1797  parser.parseType(ty) ||
1798  parser.resolveOperand(operand, ty, result.operands) ||
1799  parser.parseRParen())
1800  return failure();
1801  }
1802 
1803  Type type;
1804  if (parser.parseColon() || parser.parseType(type))
1805  return failure();
1806 
1807  result.addTypes(type);
1808  return success();
1809 }
1810 
1811 //===----------------------------------------------------------------------===//
1812 // ExtractValueOp
1813 //===----------------------------------------------------------------------===//
1814 
1815 /// Extract the type at `position` in the LLVM IR aggregate type
1816 /// `containerType`. Each element of `position` is an index into a nested
1817 /// aggregate type. Return the resulting type or emit an error.
1819  function_ref<InFlightDiagnostic(StringRef)> emitError, Type containerType,
1820  ArrayRef<int64_t> position) {
1821  Type llvmType = containerType;
1822  if (!isCompatibleType(containerType)) {
1823  emitError("expected LLVM IR Dialect type, got ") << containerType;
1824  return {};
1825  }
1826 
1827  // Infer the element type from the structure type: iteratively step inside the
1828  // type by taking the element type, indexed by the position attribute for
1829  // structures. Check the position index before accessing, it is supposed to
1830  // be in bounds.
1831  for (int64_t idx : position) {
1832  if (auto arrayType = llvm::dyn_cast<LLVMArrayType>(llvmType)) {
1833  if (idx < 0 || static_cast<unsigned>(idx) >= arrayType.getNumElements()) {
1834  emitError("position out of bounds: ") << idx;
1835  return {};
1836  }
1837  llvmType = arrayType.getElementType();
1838  } else if (auto structType = llvm::dyn_cast<LLVMStructType>(llvmType)) {
1839  if (idx < 0 ||
1840  static_cast<unsigned>(idx) >= structType.getBody().size()) {
1841  emitError("position out of bounds: ") << idx;
1842  return {};
1843  }
1844  llvmType = structType.getBody()[idx];
1845  } else {
1846  emitError("expected LLVM IR structure/array type, got: ") << llvmType;
1847  return {};
1848  }
1849  }
1850  return llvmType;
1851 }
1852 
1853 /// Extract the type at `position` in the wrapped LLVM IR aggregate type
1854 /// `containerType`.
1856  ArrayRef<int64_t> position) {
1857  for (int64_t idx : position) {
1858  if (auto structType = llvm::dyn_cast<LLVMStructType>(llvmType))
1859  llvmType = structType.getBody()[idx];
1860  else
1861  llvmType = llvm::cast<LLVMArrayType>(llvmType).getElementType();
1862  }
1863  return llvmType;
1864 }
1865 
1866 OpFoldResult LLVM::ExtractValueOp::fold(FoldAdaptor adaptor) {
1867  if (auto extractValueOp = getContainer().getDefiningOp<ExtractValueOp>()) {
1868  SmallVector<int64_t, 4> newPos(extractValueOp.getPosition());
1869  newPos.append(getPosition().begin(), getPosition().end());
1870  setPosition(newPos);
1871  getContainerMutable().set(extractValueOp.getContainer());
1872  return getResult();
1873  }
1874 
1875  {
1876  DenseElementsAttr constval;
1877  matchPattern(getContainer(), m_Constant(&constval));
1878  if (constval && constval.getElementType() == getType()) {
1879  if (isa<SplatElementsAttr>(constval))
1880  return constval.getSplatValue<Attribute>();
1881  if (getPosition().size() == 1)
1882  return constval.getValues<Attribute>()[getPosition()[0]];
1883  }
1884  }
1885 
1886  auto insertValueOp = getContainer().getDefiningOp<InsertValueOp>();
1887  OpFoldResult result = {};
1888  ArrayRef<int64_t> extractPos = getPosition();
1889  bool switchedToInsertedValue = false;
1890  while (insertValueOp) {
1891  ArrayRef<int64_t> insertPos = insertValueOp.getPosition();
1892  auto extractPosSize = extractPos.size();
1893  auto insertPosSize = insertPos.size();
1894 
1895  // Case 1: Exact match of positions.
1896  if (extractPos == insertPos)
1897  return insertValueOp.getValue();
1898 
1899  // Case 2: Insert position is a prefix of extract position. Continue
1900  // traversal with the inserted value. Example:
1901  // ```
1902  // %0 = llvm.insertvalue %arg1, %undef[0] : !llvm.struct<(i32, i32, i32)>
1903  // %1 = llvm.insertvalue %arg2, %0[1] : !llvm.struct<(i32, i32, i32)>
1904  // %2 = llvm.insertvalue %arg3, %1[2] : !llvm.struct<(i32, i32, i32)>
1905  // %3 = llvm.insertvalue %2, %foo[0]
1906  // : !llvm.struct<(struct<(i32, i32, i32)>, i64)>
1907  // %4 = llvm.extractvalue %3[0, 0]
1908  // : !llvm.struct<(struct<(i32, i32, i32)>, i64)>
1909  // ```
1910  // In the above example, %4 is folded to %arg1.
1911  if (extractPosSize > insertPosSize &&
1912  extractPos.take_front(insertPosSize) == insertPos) {
1913  insertValueOp = insertValueOp.getValue().getDefiningOp<InsertValueOp>();
1914  extractPos = extractPos.drop_front(insertPosSize);
1915  switchedToInsertedValue = true;
1916  continue;
1917  }
1918 
1919  // Case 3: Try to continue the traversal with the container value.
1920  unsigned min = std::min(extractPosSize, insertPosSize);
1921 
1922  // If one is fully prefix of the other, stop propagating back as it will
1923  // miss dependencies. For instance, %3 should not fold to %f0 in the
1924  // following example:
1925  // ```
1926  // %1 = llvm.insertvalue %f0, %0[0, 0] :
1927  // !llvm.array<4 x !llvm.array<4 x f32>>
1928  // %2 = llvm.insertvalue %arr, %1[0] :
1929  // !llvm.array<4 x !llvm.array<4 x f32>>
1930  // %3 = llvm.extractvalue %2[0, 0] : !llvm.array<4 x !llvm.array<4 x f32>>
1931  // ```
1932  if (extractPos.take_front(min) == insertPos.take_front(min))
1933  return result;
1934  // If neither a prefix, nor the exact position, we can extract out of the
1935  // value being inserted into. Moreover, we can try again if that operand
1936  // is itself an insertvalue expression.
1937  if (!switchedToInsertedValue) {
1938  // Do not swap out the container operand if we decided earlier to
1939  // continue the traversal with the inserted value (Case 2).
1940  getContainerMutable().assign(insertValueOp.getContainer());
1941  result = getResult();
1942  }
1943  insertValueOp = insertValueOp.getContainer().getDefiningOp<InsertValueOp>();
1944  }
1945  return result;
1946 }
1947 
1948 LogicalResult ExtractValueOp::verify() {
1949  auto emitError = [this](StringRef msg) { return emitOpError(msg); };
1951  emitError, getContainer().getType(), getPosition());
1952  if (!valueType)
1953  return failure();
1954 
1955  if (getRes().getType() != valueType)
1956  return emitOpError() << "Type mismatch: extracting from "
1957  << getContainer().getType() << " should produce "
1958  << valueType << " but this op returns "
1959  << getRes().getType();
1960  return success();
1961 }
1962 
1963 void ExtractValueOp::build(OpBuilder &builder, OperationState &state,
1964  Value container, ArrayRef<int64_t> position) {
1965  build(builder, state,
1966  getInsertExtractValueElementType(container.getType(), position),
1967  container, builder.getAttr<DenseI64ArrayAttr>(position));
1968 }
1969 
1970 //===----------------------------------------------------------------------===//
1971 // InsertValueOp
1972 //===----------------------------------------------------------------------===//
1973 
1974 /// Infer the value type from the container type and position.
1975 static ParseResult
1977  Type containerType,
1978  DenseI64ArrayAttr position) {
1980  [&](StringRef msg) {
1981  return parser.emitError(parser.getCurrentLocation(), msg);
1982  },
1983  containerType, position.asArrayRef());
1984  return success(!!valueType);
1985 }
1986 
1987 /// Nothing to print for an inferred type.
1989  Operation *op, Type valueType,
1990  Type containerType,
1991  DenseI64ArrayAttr position) {}
1992 
1993 LogicalResult InsertValueOp::verify() {
1994  auto emitError = [this](StringRef msg) { return emitOpError(msg); };
1996  emitError, getContainer().getType(), getPosition());
1997  if (!valueType)
1998  return failure();
1999 
2000  if (getValue().getType() != valueType)
2001  return emitOpError() << "Type mismatch: cannot insert "
2002  << getValue().getType() << " into "
2003  << getContainer().getType();
2004 
2005  return success();
2006 }
2007 
2008 //===----------------------------------------------------------------------===//
2009 // ReturnOp
2010 //===----------------------------------------------------------------------===//
2011 
2012 LogicalResult ReturnOp::verify() {
2013  auto parent = (*this)->getParentOfType<LLVMFuncOp>();
2014  if (!parent)
2015  return success();
2016 
2017  Type expectedType = parent.getFunctionType().getReturnType();
2018  if (llvm::isa<LLVMVoidType>(expectedType)) {
2019  if (!getArg())
2020  return success();
2021  InFlightDiagnostic diag = emitOpError("expected no operands");
2022  diag.attachNote(parent->getLoc()) << "when returning from function";
2023  return diag;
2024  }
2025  if (!getArg()) {
2026  if (llvm::isa<LLVMVoidType>(expectedType))
2027  return success();
2028  InFlightDiagnostic diag = emitOpError("expected 1 operand");
2029  diag.attachNote(parent->getLoc()) << "when returning from function";
2030  return diag;
2031  }
2032  if (expectedType != getArg().getType()) {
2033  InFlightDiagnostic diag = emitOpError("mismatching result types");
2034  diag.attachNote(parent->getLoc()) << "when returning from function";
2035  return diag;
2036  }
2037  return success();
2038 }
2039 
2040 //===----------------------------------------------------------------------===//
2041 // LLVM::AddressOfOp.
2042 //===----------------------------------------------------------------------===//
2043 
2045  Operation *module = op->getParentOp();
2046  while (module && !satisfiesLLVMModule(module))
2047  module = module->getParentOp();
2048  assert(module && "unexpected operation outside of a module");
2049  return module;
2050 }
2051 
2052 GlobalOp AddressOfOp::getGlobal(SymbolTableCollection &symbolTable) {
2053  return dyn_cast_or_null<GlobalOp>(
2054  symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
2055 }
2056 
2057 LLVMFuncOp AddressOfOp::getFunction(SymbolTableCollection &symbolTable) {
2058  return dyn_cast_or_null<LLVMFuncOp>(
2059  symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
2060 }
2061 
2062 AliasOp AddressOfOp::getAlias(SymbolTableCollection &symbolTable) {
2063  return dyn_cast_or_null<AliasOp>(
2064  symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
2065 }
2066 
2067 LogicalResult
2068 AddressOfOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2069  Operation *symbol =
2070  symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr());
2071 
2072  auto global = dyn_cast_or_null<GlobalOp>(symbol);
2073  auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
2074  auto alias = dyn_cast_or_null<AliasOp>(symbol);
2075 
2076  if (!global && !function && !alias)
2077  return emitOpError("must reference a global defined by 'llvm.mlir.global', "
2078  "'llvm.mlir.alias' or 'llvm.func'");
2079 
2080  LLVMPointerType type = getType();
2081  if ((global && global.getAddrSpace() != type.getAddressSpace()) ||
2082  (alias && alias.getAddrSpace() != type.getAddressSpace()))
2083  return emitOpError("pointer address space must match address space of the "
2084  "referenced global or alias");
2085 
2086  return success();
2087 }
2088 
2089 // AddressOfOp constant-folds to the global symbol name.
2090 OpFoldResult LLVM::AddressOfOp::fold(FoldAdaptor) {
2091  return getGlobalNameAttr();
2092 }
2093 
2094 //===----------------------------------------------------------------------===//
2095 // LLVM::DSOLocalEquivalentOp
2096 //===----------------------------------------------------------------------===//
2097 
2098 LLVMFuncOp
2099 DSOLocalEquivalentOp::getFunction(SymbolTableCollection &symbolTable) {
2100  return dyn_cast_or_null<LLVMFuncOp>(symbolTable.lookupSymbolIn(
2101  parentLLVMModule(*this), getFunctionNameAttr()));
2102 }
2103 
2104 AliasOp DSOLocalEquivalentOp::getAlias(SymbolTableCollection &symbolTable) {
2105  return dyn_cast_or_null<AliasOp>(symbolTable.lookupSymbolIn(
2106  parentLLVMModule(*this), getFunctionNameAttr()));
2107 }
2108 
2109 LogicalResult
2110 DSOLocalEquivalentOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2111  Operation *symbol = symbolTable.lookupSymbolIn(parentLLVMModule(*this),
2112  getFunctionNameAttr());
2113  auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
2114  auto alias = dyn_cast_or_null<AliasOp>(symbol);
2115 
2116  if (!function && !alias)
2117  return emitOpError(
2118  "must reference a global defined by 'llvm.func' or 'llvm.mlir.alias'");
2119 
2120  if (alias) {
2121  if (alias.getInitializer()
2122  .walk([&](AddressOfOp addrOp) {
2123  if (addrOp.getGlobal(symbolTable))
2124  return WalkResult::interrupt();
2125  return WalkResult::advance();
2126  })
2127  .wasInterrupted())
2128  return emitOpError("must reference an alias to a function");
2129  }
2130 
2131  if ((function && function.getLinkage() == LLVM::Linkage::ExternWeak) ||
2132  (alias && alias.getLinkage() == LLVM::Linkage::ExternWeak))
2133  return emitOpError(
2134  "target function with 'extern_weak' linkage not allowed");
2135 
2136  return success();
2137 }
2138 
2139 /// Fold a dso_local_equivalent operation to a dedicated dso_local_equivalent
2140 /// attribute.
2141 OpFoldResult DSOLocalEquivalentOp::fold(FoldAdaptor) {
2142  return DSOLocalEquivalentAttr::get(getContext(), getFunctionNameAttr());
2143 }
2144 
2145 //===----------------------------------------------------------------------===//
2146 // Verifier for LLVM::ComdatOp.
2147 //===----------------------------------------------------------------------===//
2148 
2149 void ComdatOp::build(OpBuilder &builder, OperationState &result,
2150  StringRef symName) {
2151  result.addAttribute(getSymNameAttrName(result.name),
2152  builder.getStringAttr(symName));
2153  Region *body = result.addRegion();
2154  body->emplaceBlock();
2155 }
2156 
2157 LogicalResult ComdatOp::verifyRegions() {
2158  Region &body = getBody();
2159  for (Operation &op : body.getOps())
2160  if (!isa<ComdatSelectorOp>(op))
2161  return op.emitError(
2162  "only comdat selector symbols can appear in a comdat region");
2163 
2164  return success();
2165 }
2166 
2167 //===----------------------------------------------------------------------===//
2168 // Builder, printer and verifier for LLVM::GlobalOp.
2169 //===----------------------------------------------------------------------===//
2170 
2171 void GlobalOp::build(OpBuilder &builder, OperationState &result, Type type,
2172  bool isConstant, Linkage linkage, StringRef name,
2173  Attribute value, uint64_t alignment, unsigned addrSpace,
2174  bool dsoLocal, bool threadLocal, SymbolRefAttr comdat,
2176  ArrayRef<Attribute> dbgExprs) {
2177  result.addAttribute(getSymNameAttrName(result.name),
2178  builder.getStringAttr(name));
2179  result.addAttribute(getGlobalTypeAttrName(result.name), TypeAttr::get(type));
2180  if (isConstant)
2181  result.addAttribute(getConstantAttrName(result.name),
2182  builder.getUnitAttr());
2183  if (value)
2184  result.addAttribute(getValueAttrName(result.name), value);
2185  if (dsoLocal)
2186  result.addAttribute(getDsoLocalAttrName(result.name),
2187  builder.getUnitAttr());
2188  if (threadLocal)
2189  result.addAttribute(getThreadLocal_AttrName(result.name),
2190  builder.getUnitAttr());
2191  if (comdat)
2192  result.addAttribute(getComdatAttrName(result.name), comdat);
2193 
2194  // Only add an alignment attribute if the "alignment" input
2195  // is different from 0. The value must also be a power of two, but
2196  // this is tested in GlobalOp::verify, not here.
2197  if (alignment != 0)
2198  result.addAttribute(getAlignmentAttrName(result.name),
2199  builder.getI64IntegerAttr(alignment));
2200 
2201  result.addAttribute(getLinkageAttrName(result.name),
2202  LinkageAttr::get(builder.getContext(), linkage));
2203  if (addrSpace != 0)
2204  result.addAttribute(getAddrSpaceAttrName(result.name),
2205  builder.getI32IntegerAttr(addrSpace));
2206  result.attributes.append(attrs.begin(), attrs.end());
2207 
2208  if (!dbgExprs.empty())
2209  result.addAttribute(getDbgExprsAttrName(result.name),
2210  ArrayAttr::get(builder.getContext(), dbgExprs));
2211 
2212  result.addRegion();
2213 }
2214 
2215 void GlobalOp::print(OpAsmPrinter &p) {
2216  p << ' ' << stringifyLinkage(getLinkage()) << ' ';
2217  StringRef visibility = stringifyVisibility(getVisibility_());
2218  if (!visibility.empty())
2219  p << visibility << ' ';
2220  if (getThreadLocal_())
2221  p << "thread_local ";
2222  if (auto unnamedAddr = getUnnamedAddr()) {
2223  StringRef str = stringifyUnnamedAddr(*unnamedAddr);
2224  if (!str.empty())
2225  p << str << ' ';
2226  }
2227  if (getConstant())
2228  p << "constant ";
2229  p.printSymbolName(getSymName());
2230  p << '(';
2231  if (auto value = getValueOrNull())
2232  p.printAttribute(value);
2233  p << ')';
2234  if (auto comdat = getComdat())
2235  p << " comdat(" << *comdat << ')';
2236 
2237  // Note that the alignment attribute is printed using the
2238  // default syntax here, even though it is an inherent attribute
2239  // (as defined in https://mlir.llvm.org/docs/LangRef/#attributes)
2240  p.printOptionalAttrDict((*this)->getAttrs(),
2241  {SymbolTable::getSymbolAttrName(),
2242  getGlobalTypeAttrName(), getConstantAttrName(),
2243  getValueAttrName(), getLinkageAttrName(),
2244  getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
2245  getVisibility_AttrName(), getComdatAttrName(),
2246  getUnnamedAddrAttrName()});
2247 
2248  // Print the trailing type unless it's a string global.
2249  if (llvm::dyn_cast_or_null<StringAttr>(getValueOrNull()))
2250  return;
2251  p << " : " << getType();
2252 
2253  Region &initializer = getInitializerRegion();
2254  if (!initializer.empty()) {
2255  p << ' ';
2256  p.printRegion(initializer, /*printEntryBlockArgs=*/false);
2257  }
2258 }
2259 
2260 static LogicalResult verifyComdat(Operation *op,
2261  std::optional<SymbolRefAttr> attr) {
2262  if (!attr)
2263  return success();
2264 
2265  auto *comdatSelector = SymbolTable::lookupNearestSymbolFrom(op, *attr);
2266  if (!isa_and_nonnull<ComdatSelectorOp>(comdatSelector))
2267  return op->emitError() << "expected comdat symbol";
2268 
2269  return success();
2270 }
2271 
2272 static LogicalResult verifyBlockTags(LLVMFuncOp funcOp) {
2273  llvm::DenseSet<BlockTagAttr> blockTags;
2274  // Note that presence of `BlockTagOp`s currently can't prevent an unrecheable
2275  // block to be removed by canonicalizer's region simplify pass, which needs to
2276  // be dialect aware to allow extra constraints to be described.
2277  WalkResult res = funcOp.walk([&](BlockTagOp blockTagOp) {
2278  if (blockTags.contains(blockTagOp.getTag())) {
2279  blockTagOp.emitError()
2280  << "duplicate block tag '" << blockTagOp.getTag().getId()
2281  << "' in the same function: ";
2282  return WalkResult::interrupt();
2283  }
2284  blockTags.insert(blockTagOp.getTag());
2285  return WalkResult::advance();
2286  });
2287 
2288  return failure(res.wasInterrupted());
2289 }
2290 
2291 /// Parse common attributes that might show up in the same order in both
2292 /// GlobalOp and AliasOp.
2293 template <typename OpType>
2294 static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser,
2295  OperationState &result) {
2296  MLIRContext *ctx = parser.getContext();
2297  // Parse optional linkage, default to External.
2298  result.addAttribute(OpType::getLinkageAttrName(result.name),
2300  ctx, parseOptionalLLVMKeyword<Linkage>(
2301  parser, result, LLVM::Linkage::External)));
2302 
2303  // Parse optional visibility, default to Default.
2304  result.addAttribute(OpType::getVisibility_AttrName(result.name),
2305  parser.getBuilder().getI64IntegerAttr(
2306  parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
2307  parser, result, LLVM::Visibility::Default)));
2308 
2309  // Parse optional UnnamedAddr, default to None.
2310  result.addAttribute(OpType::getUnnamedAddrAttrName(result.name),
2311  parser.getBuilder().getI64IntegerAttr(
2312  parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
2313  parser, result, LLVM::UnnamedAddr::None)));
2314 
2315  if (succeeded(parser.parseOptionalKeyword("thread_local")))
2316  result.addAttribute(OpType::getThreadLocal_AttrName(result.name),
2317  parser.getBuilder().getUnitAttr());
2318 
2319  return success();
2320 }
2321 
2322 // operation ::= `llvm.mlir.global` linkage? visibility?
2323 // (`unnamed_addr` | `local_unnamed_addr`)?
2324 // `thread_local`? `constant`? `@` identifier
2325 // `(` attribute? `)` (`comdat(` symbol-ref-id `)`)?
2326 // attribute-list? (`:` type)? region?
2327 //
2328 // The type can be omitted for string attributes, in which case it will be
2329 // inferred from the value of the string as [strlen(value) x i8].
2330 ParseResult GlobalOp::parse(OpAsmParser &parser, OperationState &result) {
2331  // Call into common parsing between GlobalOp and AliasOp.
2332  if (parseCommonGlobalAndAlias<GlobalOp>(parser, result).failed())
2333  return failure();
2334 
2335  if (succeeded(parser.parseOptionalKeyword("constant")))
2336  result.addAttribute(getConstantAttrName(result.name),
2337  parser.getBuilder().getUnitAttr());
2338 
2339  StringAttr name;
2340  if (parser.parseSymbolName(name, getSymNameAttrName(result.name),
2341  result.attributes) ||
2342  parser.parseLParen())
2343  return failure();
2344 
2345  Attribute value;
2346  if (parser.parseOptionalRParen()) {
2347  if (parser.parseAttribute(value, getValueAttrName(result.name),
2348  result.attributes) ||
2349  parser.parseRParen())
2350  return failure();
2351  }
2352 
2353  if (succeeded(parser.parseOptionalKeyword("comdat"))) {
2354  SymbolRefAttr comdat;
2355  if (parser.parseLParen() || parser.parseAttribute(comdat) ||
2356  parser.parseRParen())
2357  return failure();
2358 
2359  result.addAttribute(getComdatAttrName(result.name), comdat);
2360  }
2361 
2362  SmallVector<Type, 1> types;
2363  if (parser.parseOptionalAttrDict(result.attributes) ||
2364  parser.parseOptionalColonTypeList(types))
2365  return failure();
2366 
2367  if (types.size() > 1)
2368  return parser.emitError(parser.getNameLoc(), "expected zero or one type");
2369 
2370  Region &initRegion = *result.addRegion();
2371  if (types.empty()) {
2372  if (auto strAttr = llvm::dyn_cast_or_null<StringAttr>(value)) {
2373  MLIRContext *context = parser.getContext();
2374  auto arrayType = LLVM::LLVMArrayType::get(IntegerType::get(context, 8),
2375  strAttr.getValue().size());
2376  types.push_back(arrayType);
2377  } else {
2378  return parser.emitError(parser.getNameLoc(),
2379  "type can only be omitted for string globals");
2380  }
2381  } else {
2382  OptionalParseResult parseResult =
2383  parser.parseOptionalRegion(initRegion, /*arguments=*/{},
2384  /*argTypes=*/{});
2385  if (parseResult.has_value() && failed(*parseResult))
2386  return failure();
2387  }
2388 
2389  result.addAttribute(getGlobalTypeAttrName(result.name),
2390  TypeAttr::get(types[0]));
2391  return success();
2392 }
2393 
2394 static bool isZeroAttribute(Attribute value) {
2395  if (auto intValue = llvm::dyn_cast<IntegerAttr>(value))
2396  return intValue.getValue().isZero();
2397  if (auto fpValue = llvm::dyn_cast<FloatAttr>(value))
2398  return fpValue.getValue().isZero();
2399  if (auto splatValue = llvm::dyn_cast<SplatElementsAttr>(value))
2400  return isZeroAttribute(splatValue.getSplatValue<Attribute>());
2401  if (auto elementsValue = llvm::dyn_cast<ElementsAttr>(value))
2402  return llvm::all_of(elementsValue.getValues<Attribute>(), isZeroAttribute);
2403  if (auto arrayValue = llvm::dyn_cast<ArrayAttr>(value))
2404  return llvm::all_of(arrayValue.getValue(), isZeroAttribute);
2405  return false;
2406 }
2407 
2408 LogicalResult GlobalOp::verify() {
2409  bool validType = isCompatibleOuterType(getType())
2410  ? !llvm::isa<LLVMVoidType, LLVMTokenType,
2411  LLVMMetadataType, LLVMLabelType>(getType())
2412  : llvm::isa<PointerElementTypeInterface>(getType());
2413  if (!validType)
2414  return emitOpError(
2415  "expects type to be a valid element type for an LLVM global");
2416  if ((*this)->getParentOp() && !satisfiesLLVMModule((*this)->getParentOp()))
2417  return emitOpError("must appear at the module level");
2418 
2419  if (auto strAttr = llvm::dyn_cast_or_null<StringAttr>(getValueOrNull())) {
2420  auto type = llvm::dyn_cast<LLVMArrayType>(getType());
2421  IntegerType elementType =
2422  type ? llvm::dyn_cast<IntegerType>(type.getElementType()) : nullptr;
2423  if (!elementType || elementType.getWidth() != 8 ||
2424  type.getNumElements() != strAttr.getValue().size())
2425  return emitOpError(
2426  "requires an i8 array type of the length equal to that of the string "
2427  "attribute");
2428  }
2429 
2430  if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType())) {
2431  if (!targetExtType.hasProperty(LLVMTargetExtType::CanBeGlobal))
2432  return emitOpError()
2433  << "this target extension type cannot be used in a global";
2434 
2435  if (Attribute value = getValueOrNull())
2436  return emitOpError() << "global with target extension type can only be "
2437  "initialized with zero-initializer";
2438  }
2439 
2440  if (getLinkage() == Linkage::Common) {
2441  if (Attribute value = getValueOrNull()) {
2442  if (!isZeroAttribute(value)) {
2443  return emitOpError()
2444  << "expected zero value for '"
2445  << stringifyLinkage(Linkage::Common) << "' linkage";
2446  }
2447  }
2448  }
2449 
2450  if (getLinkage() == Linkage::Appending) {
2451  if (!llvm::isa<LLVMArrayType>(getType())) {
2452  return emitOpError() << "expected array type for '"
2453  << stringifyLinkage(Linkage::Appending)
2454  << "' linkage";
2455  }
2456  }
2457 
2458  if (failed(verifyComdat(*this, getComdat())))
2459  return failure();
2460 
2461  std::optional<uint64_t> alignAttr = getAlignment();
2462  if (alignAttr.has_value()) {
2463  uint64_t value = alignAttr.value();
2464  if (!llvm::isPowerOf2_64(value))
2465  return emitError() << "alignment attribute is not a power of 2";
2466  }
2467 
2468  return success();
2469 }
2470 
2471 LogicalResult GlobalOp::verifyRegions() {
2472  if (Block *b = getInitializerBlock()) {
2473  ReturnOp ret = cast<ReturnOp>(b->getTerminator());
2474  if (ret.operand_type_begin() == ret.operand_type_end())
2475  return emitOpError("initializer region cannot return void");
2476  if (*ret.operand_type_begin() != getType())
2477  return emitOpError("initializer region type ")
2478  << *ret.operand_type_begin() << " does not match global type "
2479  << getType();
2480 
2481  for (Operation &op : *b) {
2482  auto iface = dyn_cast<MemoryEffectOpInterface>(op);
2483  if (!iface || !iface.hasNoEffect())
2484  return op.emitError()
2485  << "ops with side effects not allowed in global initializers";
2486  }
2487 
2488  if (getValueOrNull())
2489  return emitOpError("cannot have both initializer value and region");
2490  }
2491 
2492  return success();
2493 }
2494 
2495 //===----------------------------------------------------------------------===//
2496 // LLVM::GlobalCtorsOp
2497 //===----------------------------------------------------------------------===//
2498 
2499 LogicalResult checkGlobalXtorData(Operation *op, ArrayAttr data) {
2500  if (data.empty())
2501  return success();
2502 
2503  if (llvm::all_of(data.getAsRange<Attribute>(), [](Attribute v) {
2504  return isa<FlatSymbolRefAttr, ZeroAttr>(v);
2505  }))
2506  return success();
2507  return op->emitError("data element must be symbol or #llvm.zero");
2508 }
2509 
2510 LogicalResult
2511 GlobalCtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2512  for (Attribute ctor : getCtors()) {
2513  if (failed(verifySymbolAttrUse(llvm::cast<FlatSymbolRefAttr>(ctor), *this,
2514  symbolTable)))
2515  return failure();
2516  }
2517  return success();
2518 }
2519 
2520 LogicalResult GlobalCtorsOp::verify() {
2521  if (checkGlobalXtorData(*this, getData()).failed())
2522  return failure();
2523 
2524  if (getCtors().size() == getPriorities().size() &&
2525  getCtors().size() == getData().size())
2526  return success();
2527  return emitError(
2528  "ctors, priorities, and data must have the same number of elements");
2529 }
2530 
2531 //===----------------------------------------------------------------------===//
2532 // LLVM::GlobalDtorsOp
2533 //===----------------------------------------------------------------------===//
2534 
2535 LogicalResult
2536 GlobalDtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2537  for (Attribute dtor : getDtors()) {
2538  if (failed(verifySymbolAttrUse(llvm::cast<FlatSymbolRefAttr>(dtor), *this,
2539  symbolTable)))
2540  return failure();
2541  }
2542  return success();
2543 }
2544 
2545 LogicalResult GlobalDtorsOp::verify() {
2546  if (checkGlobalXtorData(*this, getData()).failed())
2547  return failure();
2548 
2549  if (getDtors().size() == getPriorities().size() &&
2550  getDtors().size() == getData().size())
2551  return success();
2552  return emitError(
2553  "dtors, priorities, and data must have the same number of elements");
2554 }
2555 
2556 //===----------------------------------------------------------------------===//
2557 // Builder, printer and verifier for LLVM::AliasOp.
2558 //===----------------------------------------------------------------------===//
2559 
2560 void AliasOp::build(OpBuilder &builder, OperationState &result, Type type,
2561  Linkage linkage, StringRef name, bool dsoLocal,
2562  bool threadLocal, ArrayRef<NamedAttribute> attrs) {
2563  result.addAttribute(getSymNameAttrName(result.name),
2564  builder.getStringAttr(name));
2565  result.addAttribute(getAliasTypeAttrName(result.name), TypeAttr::get(type));
2566  if (dsoLocal)
2567  result.addAttribute(getDsoLocalAttrName(result.name),
2568  builder.getUnitAttr());
2569  if (threadLocal)
2570  result.addAttribute(getThreadLocal_AttrName(result.name),
2571  builder.getUnitAttr());
2572 
2573  result.addAttribute(getLinkageAttrName(result.name),
2574  LinkageAttr::get(builder.getContext(), linkage));
2575  result.attributes.append(attrs.begin(), attrs.end());
2576 
2577  result.addRegion();
2578 }
2579 
2580 void AliasOp::print(OpAsmPrinter &p) {
2581  p << ' ' << stringifyLinkage(getLinkage()) << ' ';
2582  StringRef visibility = stringifyVisibility(getVisibility_());
2583  if (!visibility.empty())
2584  p << visibility << ' ';
2585 
2586  if (std::optional<mlir::LLVM::UnnamedAddr> unnamedAddr = getUnnamedAddr()) {
2587  StringRef str = stringifyUnnamedAddr(*unnamedAddr);
2588  if (!str.empty())
2589  p << str << ' ';
2590  }
2591 
2592  if (getThreadLocal_())
2593  p << "thread_local ";
2594 
2595  p.printSymbolName(getSymName());
2596  p.printOptionalAttrDict((*this)->getAttrs(),
2597  {SymbolTable::getSymbolAttrName(),
2598  getAliasTypeAttrName(), getLinkageAttrName(),
2599  getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
2600  getVisibility_AttrName(), getUnnamedAddrAttrName()});
2601 
2602  // Print the trailing type.
2603  p << " : " << getType() << ' ';
2604  // Print the initializer region.
2605  p.printRegion(getInitializerRegion(), /*printEntryBlockArgs=*/false);
2606 }
2607 
2608 // operation ::= `llvm.mlir.alias` linkage? visibility?
2609 // (`unnamed_addr` | `local_unnamed_addr`)?
2610 // `thread_local`? `@` identifier
2611 // `(` attribute? `)`
2612 // attribute-list? `:` type region
2613 //
2614 ParseResult AliasOp::parse(OpAsmParser &parser, OperationState &result) {
2615  // Call into common parsing between GlobalOp and AliasOp.
2616  if (parseCommonGlobalAndAlias<AliasOp>(parser, result).failed())
2617  return failure();
2618 
2619  StringAttr name;
2620  if (parser.parseSymbolName(name, getSymNameAttrName(result.name),
2621  result.attributes))
2622  return failure();
2623 
2624  SmallVector<Type, 1> types;
2625  if (parser.parseOptionalAttrDict(result.attributes) ||
2626  parser.parseOptionalColonTypeList(types))
2627  return failure();
2628 
2629  if (types.size() > 1)
2630  return parser.emitError(parser.getNameLoc(), "expected zero or one type");
2631 
2632  Region &initRegion = *result.addRegion();
2633  if (parser.parseRegion(initRegion).failed())
2634  return failure();
2635 
2636  result.addAttribute(getAliasTypeAttrName(result.name),
2637  TypeAttr::get(types[0]));
2638  return success();
2639 }
2640 
2641 LogicalResult AliasOp::verify() {
2642  bool validType = isCompatibleOuterType(getType())
2643  ? !llvm::isa<LLVMVoidType, LLVMTokenType,
2644  LLVMMetadataType, LLVMLabelType>(getType())
2645  : llvm::isa<PointerElementTypeInterface>(getType());
2646  if (!validType)
2647  return emitOpError(
2648  "expects type to be a valid element type for an LLVM global alias");
2649 
2650  // This matches LLVM IR verification logic, see llvm/lib/IR/Verifier.cpp
2651  switch (getLinkage()) {
2652  case Linkage::External:
2653  case Linkage::Internal:
2654  case Linkage::Private:
2655  case Linkage::Weak:
2656  case Linkage::WeakODR:
2657  case Linkage::Linkonce:
2658  case Linkage::LinkonceODR:
2659  case Linkage::AvailableExternally:
2660  break;
2661  default:
2662  return emitOpError()
2663  << "'" << stringifyLinkage(getLinkage())
2664  << "' linkage not supported in aliases, available options: private, "
2665  "internal, linkonce, weak, linkonce_odr, weak_odr, external or "
2666  "available_externally";
2667  }
2668 
2669  return success();
2670 }
2671 
2672 LogicalResult AliasOp::verifyRegions() {
2673  Block &b = getInitializerBlock();
2674  auto ret = cast<ReturnOp>(b.getTerminator());
2675  if (ret.getNumOperands() == 0 ||
2676  !isa<LLVM::LLVMPointerType>(ret.getOperand(0).getType()))
2677  return emitOpError("initializer region must always return a pointer");
2678 
2679  for (Operation &op : b) {
2680  auto iface = dyn_cast<MemoryEffectOpInterface>(op);
2681  if (!iface || !iface.hasNoEffect())
2682  return op.emitError()
2683  << "ops with side effects are not allowed in alias initializers";
2684  }
2685 
2686  return success();
2687 }
2688 
2689 unsigned AliasOp::getAddrSpace() {
2690  Block &initializer = getInitializerBlock();
2691  auto ret = cast<ReturnOp>(initializer.getTerminator());
2692  auto ptrTy = cast<LLVMPointerType>(ret.getOperand(0).getType());
2693  return ptrTy.getAddressSpace();
2694 }
2695 
2696 //===----------------------------------------------------------------------===//
2697 // ShuffleVectorOp
2698 //===----------------------------------------------------------------------===//
2699 
2700 void ShuffleVectorOp::build(OpBuilder &builder, OperationState &state, Value v1,
2701  Value v2, DenseI32ArrayAttr mask,
2702  ArrayRef<NamedAttribute> attrs) {
2703  auto containerType = v1.getType();
2704  auto vType = LLVM::getVectorType(
2705  cast<VectorType>(containerType).getElementType(), mask.size(),
2706  LLVM::isScalableVectorType(containerType));
2707  build(builder, state, vType, v1, v2, mask);
2708  state.addAttributes(attrs);
2709 }
2710 
2711 void ShuffleVectorOp::build(OpBuilder &builder, OperationState &state, Value v1,
2712  Value v2, ArrayRef<int32_t> mask) {
2713  build(builder, state, v1, v2, builder.getDenseI32ArrayAttr(mask));
2714 }
2715 
2716 /// Build the result type of a shuffle vector operation.
2717 static ParseResult parseShuffleType(AsmParser &parser, Type v1Type,
2718  Type &resType, DenseI32ArrayAttr mask) {
2719  if (!LLVM::isCompatibleVectorType(v1Type))
2720  return parser.emitError(parser.getCurrentLocation(),
2721  "expected an LLVM compatible vector type");
2722  resType =
2723  LLVM::getVectorType(cast<VectorType>(v1Type).getElementType(),
2724  mask.size(), LLVM::isScalableVectorType(v1Type));
2725  return success();
2726 }
2727 
2728 /// Nothing to do when the result type is inferred.
2729 static void printShuffleType(AsmPrinter &printer, Operation *op, Type v1Type,
2730  Type resType, DenseI32ArrayAttr mask) {}
2731 
2732 LogicalResult ShuffleVectorOp::verify() {
2733  if (LLVM::isScalableVectorType(getV1().getType()) &&
2734  llvm::any_of(getMask(), [](int32_t v) { return v != 0; }))
2735  return emitOpError("expected a splat operation for scalable vectors");
2736  return success();
2737 }
2738 
2739 //===----------------------------------------------------------------------===//
2740 // Implementations for LLVM::LLVMFuncOp.
2741 //===----------------------------------------------------------------------===//
2742 
2743 // Add the entry block to the function.
2744 Block *LLVMFuncOp::addEntryBlock(OpBuilder &builder) {
2745  assert(empty() && "function already has an entry block");
2746  OpBuilder::InsertionGuard g(builder);
2747  Block *entry = builder.createBlock(&getBody());
2748 
2749  // FIXME: Allow passing in proper locations for the entry arguments.
2750  LLVMFunctionType type = getFunctionType();
2751  for (unsigned i = 0, e = type.getNumParams(); i < e; ++i)
2752  entry->addArgument(type.getParamType(i), getLoc());
2753  return entry;
2754 }
2755 
2756 void LLVMFuncOp::build(OpBuilder &builder, OperationState &result,
2757  StringRef name, Type type, LLVM::Linkage linkage,
2758  bool dsoLocal, CConv cconv, SymbolRefAttr comdat,
2760  ArrayRef<DictionaryAttr> argAttrs,
2761  std::optional<uint64_t> functionEntryCount) {
2762  result.addRegion();
2764  builder.getStringAttr(name));
2765  result.addAttribute(getFunctionTypeAttrName(result.name),
2766  TypeAttr::get(type));
2767  result.addAttribute(getLinkageAttrName(result.name),
2768  LinkageAttr::get(builder.getContext(), linkage));
2769  result.addAttribute(getCConvAttrName(result.name),
2770  CConvAttr::get(builder.getContext(), cconv));
2771  result.attributes.append(attrs.begin(), attrs.end());
2772  if (dsoLocal)
2773  result.addAttribute(getDsoLocalAttrName(result.name),
2774  builder.getUnitAttr());
2775  if (comdat)
2776  result.addAttribute(getComdatAttrName(result.name), comdat);
2777  if (functionEntryCount)
2778  result.addAttribute(getFunctionEntryCountAttrName(result.name),
2779  builder.getI64IntegerAttr(functionEntryCount.value()));
2780  if (argAttrs.empty())
2781  return;
2782 
2783  assert(llvm::cast<LLVMFunctionType>(type).getNumParams() == argAttrs.size() &&
2784  "expected as many argument attribute lists as arguments");
2786  builder, result, argAttrs, /*resultAttrs=*/std::nullopt,
2787  getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
2788 }
2789 
2790 // Builds an LLVM function type from the given lists of input and output types.
2791 // Returns a null type if any of the types provided are non-LLVM types, or if
2792 // there is more than one output type.
2793 static Type
2795  ArrayRef<Type> outputs,
2797  Builder &b = parser.getBuilder();
2798  if (outputs.size() > 1) {
2799  parser.emitError(loc, "failed to construct function type: expected zero or "
2800  "one function result");
2801  return {};
2802  }
2803 
2804  // Convert inputs to LLVM types, exit early on error.
2805  SmallVector<Type, 4> llvmInputs;
2806  for (auto t : inputs) {
2807  if (!isCompatibleType(t)) {
2808  parser.emitError(loc, "failed to construct function type: expected LLVM "
2809  "type for function arguments");
2810  return {};
2811  }
2812  llvmInputs.push_back(t);
2813  }
2814 
2815  // No output is denoted as "void" in LLVM type system.
2816  Type llvmOutput =
2817  outputs.empty() ? LLVMVoidType::get(b.getContext()) : outputs.front();
2818  if (!isCompatibleType(llvmOutput)) {
2819  parser.emitError(loc, "failed to construct function type: expected LLVM "
2820  "type for function results")
2821  << llvmOutput;
2822  return {};
2823  }
2824  return LLVMFunctionType::get(llvmOutput, llvmInputs,
2825  variadicFlag.isVariadic());
2826 }
2827 
2828 // Parses an LLVM function.
2829 //
2830 // operation ::= `llvm.func` linkage? cconv? function-signature
2831 // (`comdat(` symbol-ref-id `)`)?
2832 // function-attributes?
2833 // function-body
2834 //
2835 ParseResult LLVMFuncOp::parse(OpAsmParser &parser, OperationState &result) {
2836  // Default to external linkage if no keyword is provided.
2837  result.addAttribute(
2838  getLinkageAttrName(result.name),
2839  LinkageAttr::get(parser.getContext(),
2840  parseOptionalLLVMKeyword<Linkage>(
2841  parser, result, LLVM::Linkage::External)));
2842 
2843  // Parse optional visibility, default to Default.
2844  result.addAttribute(getVisibility_AttrName(result.name),
2845  parser.getBuilder().getI64IntegerAttr(
2846  parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
2847  parser, result, LLVM::Visibility::Default)));
2848 
2849  // Parse optional UnnamedAddr, default to None.
2850  result.addAttribute(getUnnamedAddrAttrName(result.name),
2851  parser.getBuilder().getI64IntegerAttr(
2852  parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
2853  parser, result, LLVM::UnnamedAddr::None)));
2854 
2855  // Default to C Calling Convention if no keyword is provided.
2856  result.addAttribute(
2857  getCConvAttrName(result.name),
2858  CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
2859  parser, result, LLVM::CConv::C)));
2860 
2861  StringAttr nameAttr;
2863  SmallVector<DictionaryAttr> resultAttrs;
2864  SmallVector<Type> resultTypes;
2865  bool isVariadic;
2866 
2867  auto signatureLocation = parser.getCurrentLocation();
2868  if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
2869  result.attributes) ||
2871  parser, /*allowVariadic=*/true, entryArgs, isVariadic, resultTypes,
2872  resultAttrs))
2873  return failure();
2874 
2875  SmallVector<Type> argTypes;
2876  for (auto &arg : entryArgs)
2877  argTypes.push_back(arg.type);
2878  auto type =
2879  buildLLVMFunctionType(parser, signatureLocation, argTypes, resultTypes,
2881  if (!type)
2882  return failure();
2883  result.addAttribute(getFunctionTypeAttrName(result.name),
2884  TypeAttr::get(type));
2885 
2886  if (succeeded(parser.parseOptionalKeyword("vscale_range"))) {
2887  int64_t minRange, maxRange;
2888  if (parser.parseLParen() || parser.parseInteger(minRange) ||
2889  parser.parseComma() || parser.parseInteger(maxRange) ||
2890  parser.parseRParen())
2891  return failure();
2892  auto intTy = IntegerType::get(parser.getContext(), 32);
2893  result.addAttribute(
2894  getVscaleRangeAttrName(result.name),
2896  IntegerAttr::get(intTy, minRange),
2897  IntegerAttr::get(intTy, maxRange)));
2898  }
2899  // Parse the optional comdat selector.
2900  if (succeeded(parser.parseOptionalKeyword("comdat"))) {
2901  SymbolRefAttr comdat;
2902  if (parser.parseLParen() || parser.parseAttribute(comdat) ||
2903  parser.parseRParen())
2904  return failure();
2905 
2906  result.addAttribute(getComdatAttrName(result.name), comdat);
2907  }
2908 
2909  if (failed(parser.parseOptionalAttrDictWithKeyword(result.attributes)))
2910  return failure();
2912  parser.getBuilder(), result, entryArgs, resultAttrs,
2913  getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
2914 
2915  auto *body = result.addRegion();
2916  OptionalParseResult parseResult =
2917  parser.parseOptionalRegion(*body, entryArgs);
2918  return failure(parseResult.has_value() && failed(*parseResult));
2919 }
2920 
2921 // Print the LLVMFuncOp. Collects argument and result types and passes them to
2922 // helper functions. Drops "void" result since it cannot be parsed back. Skips
2923 // the external linkage since it is the default value.
2925  p << ' ';
2926  if (getLinkage() != LLVM::Linkage::External)
2927  p << stringifyLinkage(getLinkage()) << ' ';
2928  StringRef visibility = stringifyVisibility(getVisibility_());
2929  if (!visibility.empty())
2930  p << visibility << ' ';
2931  if (auto unnamedAddr = getUnnamedAddr()) {
2932  StringRef str = stringifyUnnamedAddr(*unnamedAddr);
2933  if (!str.empty())
2934  p << str << ' ';
2935  }
2936  if (getCConv() != LLVM::CConv::C)
2937  p << stringifyCConv(getCConv()) << ' ';
2938 
2939  p.printSymbolName(getName());
2940 
2941  LLVMFunctionType fnType = getFunctionType();
2942  SmallVector<Type, 8> argTypes;
2943  SmallVector<Type, 1> resTypes;
2944  argTypes.reserve(fnType.getNumParams());
2945  for (unsigned i = 0, e = fnType.getNumParams(); i < e; ++i)
2946  argTypes.push_back(fnType.getParamType(i));
2947 
2948  Type returnType = fnType.getReturnType();
2949  if (!llvm::isa<LLVMVoidType>(returnType))
2950  resTypes.push_back(returnType);
2951 
2953  isVarArg(), resTypes);
2954 
2955  // Print vscale range if present
2956  if (std::optional<VScaleRangeAttr> vscale = getVscaleRange())
2957  p << " vscale_range(" << vscale->getMinRange().getInt() << ", "
2958  << vscale->getMaxRange().getInt() << ')';
2959 
2960  // Print the optional comdat selector.
2961  if (auto comdat = getComdat())
2962  p << " comdat(" << *comdat << ')';
2963 
2965  p, *this,
2966  {getFunctionTypeAttrName(), getArgAttrsAttrName(), getResAttrsAttrName(),
2967  getLinkageAttrName(), getCConvAttrName(), getVisibility_AttrName(),
2968  getComdatAttrName(), getUnnamedAddrAttrName(),
2969  getVscaleRangeAttrName()});
2970 
2971  // Print the body if this is not an external function.
2972  Region &body = getBody();
2973  if (!body.empty()) {
2974  p << ' ';
2975  p.printRegion(body, /*printEntryBlockArgs=*/false,
2976  /*printBlockTerminators=*/true);
2977  }
2978 }
2979 
2980 // Verifies LLVM- and implementation-specific properties of the LLVM func Op:
2981 // - functions don't have 'common' linkage
2982 // - external functions have 'external' or 'extern_weak' linkage;
2983 // - vararg is (currently) only supported for external functions;
2984 LogicalResult LLVMFuncOp::verify() {
2985  if (getLinkage() == LLVM::Linkage::Common)
2986  return emitOpError() << "functions cannot have '"
2987  << stringifyLinkage(LLVM::Linkage::Common)
2988  << "' linkage";
2989 
2990  if (failed(verifyComdat(*this, getComdat())))
2991  return failure();
2992 
2993  if (isExternal()) {
2994  if (getLinkage() != LLVM::Linkage::External &&
2995  getLinkage() != LLVM::Linkage::ExternWeak)
2996  return emitOpError() << "external functions must have '"
2997  << stringifyLinkage(LLVM::Linkage::External)
2998  << "' or '"
2999  << stringifyLinkage(LLVM::Linkage::ExternWeak)
3000  << "' linkage";
3001  return success();
3002  }
3003 
3004  // In LLVM IR, these attributes are composed by convention, not by design.
3005  if (isNoInline() && isAlwaysInline())
3006  return emitError("no_inline and always_inline attributes are incompatible");
3007 
3008  if (isOptimizeNone() && !isNoInline())
3009  return emitOpError("with optimize_none must also be no_inline");
3010 
3011  Type landingpadResultTy;
3012  StringRef diagnosticMessage;
3013  bool isLandingpadTypeConsistent =
3014  !walk([&](Operation *op) {
3015  const auto checkType = [&](Type type, StringRef errorMessage) {
3016  if (!landingpadResultTy) {
3017  landingpadResultTy = type;
3018  return WalkResult::advance();
3019  }
3020  if (landingpadResultTy != type) {
3021  diagnosticMessage = errorMessage;
3022  return WalkResult::interrupt();
3023  }
3024  return WalkResult::advance();
3025  };
3027  .Case<LandingpadOp>([&](auto landingpad) {
3028  constexpr StringLiteral errorMessage =
3029  "'llvm.landingpad' should have a consistent result type "
3030  "inside a function";
3031  return checkType(landingpad.getType(), errorMessage);
3032  })
3033  .Case<ResumeOp>([&](auto resume) {
3034  constexpr StringLiteral errorMessage =
3035  "'llvm.resume' should have a consistent input type inside a "
3036  "function";
3037  return checkType(resume.getValue().getType(), errorMessage);
3038  })
3039  .Default([](auto) { return WalkResult::skip(); });
3040  }).wasInterrupted();
3041  if (!isLandingpadTypeConsistent) {
3042  assert(!diagnosticMessage.empty() &&
3043  "Expecting a non-empty diagnostic message");
3044  return emitError(diagnosticMessage);
3045  }
3046 
3047  if (failed(verifyBlockTags(*this)))
3048  return failure();
3049 
3050  return success();
3051 }
3052 
3053 /// Verifies LLVM- and implementation-specific properties of the LLVM func Op:
3054 /// - entry block arguments are of LLVM types.
3055 LogicalResult LLVMFuncOp::verifyRegions() {
3056  if (isExternal())
3057  return success();
3058 
3059  unsigned numArguments = getFunctionType().getNumParams();
3060  Block &entryBlock = front();
3061  for (unsigned i = 0; i < numArguments; ++i) {
3062  Type argType = entryBlock.getArgument(i).getType();
3063  if (!isCompatibleType(argType))
3064  return emitOpError("entry block argument #")
3065  << i << " is not of LLVM type";
3066  }
3067 
3068  return success();
3069 }
3070 
3071 Region *LLVMFuncOp::getCallableRegion() {
3072  if (isExternal())
3073  return nullptr;
3074  return &getBody();
3075 }
3076 
3077 //===----------------------------------------------------------------------===//
3078 // UndefOp.
3079 //===----------------------------------------------------------------------===//
3080 
3081 /// Fold an undef operation to a dedicated undef attribute.
3082 OpFoldResult LLVM::UndefOp::fold(FoldAdaptor) {
3083  return LLVM::UndefAttr::get(getContext());
3084 }
3085 
3086 //===----------------------------------------------------------------------===//
3087 // PoisonOp.
3088 //===----------------------------------------------------------------------===//
3089 
3090 /// Fold a poison operation to a dedicated poison attribute.
3091 OpFoldResult LLVM::PoisonOp::fold(FoldAdaptor) {
3093 }
3094 
3095 //===----------------------------------------------------------------------===//
3096 // ZeroOp.
3097 //===----------------------------------------------------------------------===//
3098 
3099 LogicalResult LLVM::ZeroOp::verify() {
3100  if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType()))
3101  if (!targetExtType.hasProperty(LLVM::LLVMTargetExtType::HasZeroInit))
3102  return emitOpError()
3103  << "target extension type does not support zero-initializer";
3104 
3105  return success();
3106 }
3107 
3108 /// Fold a zero operation to a builtin zero attribute when possible and fall
3109 /// back to a dedicated zero attribute.
3110 OpFoldResult LLVM::ZeroOp::fold(FoldAdaptor) {
3112  if (result)
3113  return result;
3114  return LLVM::ZeroAttr::get(getContext());
3115 }
3116 
3117 //===----------------------------------------------------------------------===//
3118 // ConstantOp.
3119 //===----------------------------------------------------------------------===//
3120 
3121 /// Compute the total number of elements in the given type, also taking into
3122 /// account nested types. Supported types are `VectorType` and `LLVMArrayType`.
3123 /// Everything else is treated as a scalar.
3124 static int64_t getNumElements(Type t) {
3125  if (auto vecType = dyn_cast<VectorType>(t)) {
3126  assert(!vecType.isScalable() &&
3127  "number of elements of a scalable vector type is unknown");
3128  return vecType.getNumElements() * getNumElements(vecType.getElementType());
3129  }
3130  if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(t))
3131  return arrayType.getNumElements() *
3132  getNumElements(arrayType.getElementType());
3133  return 1;
3134 }
3135 
3136 /// Check if the given type is a scalable vector type or a vector/array type
3137 /// that contains a nested scalable vector type.
3138 static bool hasScalableVectorType(Type t) {
3139  if (auto vecType = dyn_cast<VectorType>(t)) {
3140  if (vecType.isScalable())
3141  return true;
3142  return hasScalableVectorType(vecType.getElementType());
3143  }
3144  if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(t))
3145  return hasScalableVectorType(arrayType.getElementType());
3146  return false;
3147 }
3148 
3149 LogicalResult LLVM::ConstantOp::verify() {
3150  if (StringAttr sAttr = llvm::dyn_cast<StringAttr>(getValue())) {
3151  auto arrayType = llvm::dyn_cast<LLVMArrayType>(getType());
3152  if (!arrayType || arrayType.getNumElements() != sAttr.getValue().size() ||
3153  !arrayType.getElementType().isInteger(8)) {
3154  return emitOpError() << "expected array type of "
3155  << sAttr.getValue().size()
3156  << " i8 elements for the string constant";
3157  }
3158  return success();
3159  }
3160  if (auto structType = dyn_cast<LLVMStructType>(getType())) {
3161  auto arrayAttr = dyn_cast<ArrayAttr>(getValue());
3162  if (!arrayAttr) {
3163  return emitOpError() << "expected array attribute for a struct constant";
3164  }
3165 
3166  ArrayRef<Type> elementTypes = structType.getBody();
3167  if (arrayAttr.size() != elementTypes.size()) {
3168  return emitOpError() << "expected array attribute of size "
3169  << elementTypes.size();
3170  }
3171  for (auto elementTy : elementTypes) {
3172  if (!isa<IntegerType, FloatType, LLVMPPCFP128Type>(elementTy)) {
3173  return emitOpError() << "expected struct element types to be floating "
3174  "point type or integer type";
3175  }
3176  }
3177 
3178  for (size_t i = 0; i < elementTypes.size(); ++i) {
3179  Attribute element = arrayAttr[i];
3180  if (!isa<IntegerAttr, FloatAttr>(element)) {
3181  return emitOpError()
3182  << "expected struct element attribute types to be floating "
3183  "point type or integer type";
3184  }
3185  auto elementType = cast<TypedAttr>(element).getType();
3186  if (elementType != elementTypes[i]) {
3187  return emitOpError()
3188  << "struct element at index " << i << " is of wrong type";
3189  }
3190  }
3191 
3192  return success();
3193  }
3194  if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType())) {
3195  return emitOpError() << "does not support target extension type.";
3196  }
3197 
3198  // Verification of IntegerAttr, FloatAttr, ElementsAttr, ArrayAttr.
3199  if (auto intAttr = dyn_cast<IntegerAttr>(getValue())) {
3200  if (!llvm::isa<IntegerType>(getType()))
3201  return emitOpError() << "expected integer type";
3202  } else if (auto floatAttr = dyn_cast<FloatAttr>(getValue())) {
3203  const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics();
3204  unsigned floatWidth = APFloat::getSizeInBits(sem);
3205  if (auto floatTy = dyn_cast<FloatType>(getType())) {
3206  if (floatTy.getWidth() != floatWidth) {
3207  return emitOpError() << "expected float type of width " << floatWidth;
3208  }
3209  }
3210  // See the comment for getLLVMConstant for more details about why 8-bit
3211  // floats can be represented by integers.
3212  if (isa<IntegerType>(getType()) && !getType().isInteger(floatWidth)) {
3213  return emitOpError() << "expected integer type of width " << floatWidth;
3214  }
3215  } else if (isa<ElementsAttr, ArrayAttr>(getValue())) {
3216  if (hasScalableVectorType(getType())) {
3217  // The exact number of elements of a scalable vector is unknown, so we
3218  // allow only splat attributes.
3219  auto splatElementsAttr = dyn_cast<SplatElementsAttr>(getValue());
3220  if (!splatElementsAttr)
3221  return emitOpError()
3222  << "scalable vector type requires a splat attribute";
3223  return success();
3224  }
3225  if (!isa<VectorType, LLVM::LLVMArrayType>(getType()))
3226  return emitOpError() << "expected vector or array type";
3227  // The number of elements of the attribute and the type must match.
3228  int64_t attrNumElements;
3229  if (auto elementsAttr = dyn_cast<ElementsAttr>(getValue()))
3230  attrNumElements = elementsAttr.getNumElements();
3231  else
3232  attrNumElements = cast<ArrayAttr>(getValue()).size();
3233  if (getNumElements(getType()) != attrNumElements)
3234  return emitOpError()
3235  << "type and attribute have a different number of elements: "
3236  << getNumElements(getType()) << " vs. " << attrNumElements;
3237  } else {
3238  return emitOpError()
3239  << "only supports integer, float, string or elements attributes";
3240  }
3241 
3242  return success();
3243 }
3244 
3245 bool LLVM::ConstantOp::isBuildableWith(Attribute value, Type type) {
3246  // The value's type must be the same as the provided type.
3247  auto typedAttr = dyn_cast<TypedAttr>(value);
3248  if (!typedAttr || typedAttr.getType() != type || !isCompatibleType(type))
3249  return false;
3250  // The value's type must be an LLVM compatible type.
3251  if (!isCompatibleType(type))
3252  return false;
3253  // TODO: Add support for additional attributes kinds once needed.
3254  return isa<IntegerAttr, FloatAttr, ElementsAttr>(value);
3255 }
3256 
3257 ConstantOp LLVM::ConstantOp::materialize(OpBuilder &builder, Attribute value,
3258  Type type, Location loc) {
3259  if (isBuildableWith(value, type))
3260  return builder.create<LLVM::ConstantOp>(loc, cast<TypedAttr>(value));
3261  return nullptr;
3262 }
3263 
3264 // Constant op constant-folds to its value.
3265 OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); }
3266 
3267 //===----------------------------------------------------------------------===//
3268 // AtomicRMWOp
3269 //===----------------------------------------------------------------------===//
3270 
3271 void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
3272  AtomicBinOp binOp, Value ptr, Value val,
3273  AtomicOrdering ordering, StringRef syncscope,
3274  unsigned alignment, bool isVolatile) {
3275  build(builder, state, val.getType(), binOp, ptr, val, ordering,
3276  !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
3277  alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
3278  /*access_groups=*/nullptr,
3279  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
3280 }
3281 
3282 LogicalResult AtomicRMWOp::verify() {
3283  auto valType = getVal().getType();
3284  if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub ||
3285  getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax) {
3286  if (isCompatibleVectorType(valType)) {
3287  if (isScalableVectorType(valType))
3288  return emitOpError("expected LLVM IR fixed vector type");
3289  Type elemType = llvm::cast<VectorType>(valType).getElementType();
3290  if (!isCompatibleFloatingPointType(elemType))
3291  return emitOpError(
3292  "expected LLVM IR floating point type for vector element");
3293  } else if (!isCompatibleFloatingPointType(valType)) {
3294  return emitOpError("expected LLVM IR floating point type");
3295  }
3296  } else if (getBinOp() == AtomicBinOp::xchg) {
3297  DataLayout dataLayout = DataLayout::closest(*this);
3298  if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
3299  return emitOpError("unexpected LLVM IR type for 'xchg' bin_op");
3300  } else {
3301  auto intType = llvm::dyn_cast<IntegerType>(valType);
3302  unsigned intBitWidth = intType ? intType.getWidth() : 0;
3303  if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 &&
3304  intBitWidth != 64)
3305  return emitOpError("expected LLVM IR integer type");
3306  }
3307 
3308  if (static_cast<unsigned>(getOrdering()) <
3309  static_cast<unsigned>(AtomicOrdering::monotonic))
3310  return emitOpError() << "expected at least '"
3311  << stringifyAtomicOrdering(AtomicOrdering::monotonic)
3312  << "' ordering";
3313 
3314  return success();
3315 }
3316 
3317 //===----------------------------------------------------------------------===//
3318 // AtomicCmpXchgOp
3319 //===----------------------------------------------------------------------===//
3320 
3321 /// Returns an LLVM struct type that contains a value type and a boolean type.
3322 static LLVMStructType getValAndBoolStructType(Type valType) {
3323  auto boolType = IntegerType::get(valType.getContext(), 1);
3324  return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType});
3325 }
3326 
3327 void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state,
3328  Value ptr, Value cmp, Value val,
3329  AtomicOrdering successOrdering,
3330  AtomicOrdering failureOrdering, StringRef syncscope,
3331  unsigned alignment, bool isWeak, bool isVolatile) {
3332  build(builder, state, getValAndBoolStructType(val.getType()), ptr, cmp, val,
3333  successOrdering, failureOrdering,
3334  !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
3335  alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak,
3336  isVolatile, /*access_groups=*/nullptr,
3337  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
3338 }
3339 
3340 LogicalResult AtomicCmpXchgOp::verify() {
3341  auto ptrType = llvm::cast<LLVM::LLVMPointerType>(getPtr().getType());
3342  if (!ptrType)
3343  return emitOpError("expected LLVM IR pointer type for operand #0");
3344  auto valType = getVal().getType();
3345  DataLayout dataLayout = DataLayout::closest(*this);
3346  if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
3347  return emitOpError("unexpected LLVM IR type");
3348  if (getSuccessOrdering() < AtomicOrdering::monotonic ||
3349  getFailureOrdering() < AtomicOrdering::monotonic)
3350  return emitOpError("ordering must be at least 'monotonic'");
3351  if (getFailureOrdering() == AtomicOrdering::release ||
3352  getFailureOrdering() == AtomicOrdering::acq_rel)
3353  return emitOpError("failure ordering cannot be 'release' or 'acq_rel'");
3354  return success();
3355 }
3356 
3357 //===----------------------------------------------------------------------===//
3358 // FenceOp
3359 //===----------------------------------------------------------------------===//
3360 
3361 void FenceOp::build(OpBuilder &builder, OperationState &state,
3362  AtomicOrdering ordering, StringRef syncscope) {
3363  build(builder, state, ordering,
3364  syncscope.empty() ? nullptr : builder.getStringAttr(syncscope));
3365 }
3366 
3367 LogicalResult FenceOp::verify() {
3368  if (getOrdering() == AtomicOrdering::not_atomic ||
3369  getOrdering() == AtomicOrdering::unordered ||
3370  getOrdering() == AtomicOrdering::monotonic)
3371  return emitOpError("can be given only acquire, release, acq_rel, "
3372  "and seq_cst orderings");
3373  return success();
3374 }
3375 
3376 //===----------------------------------------------------------------------===//
3377 // Verifier for extension ops
3378 //===----------------------------------------------------------------------===//
3379 
3380 /// Verifies that the given extension operation operates on consistent scalars
3381 /// or vectors, and that the target width is larger than the input width.
3382 template <class ExtOp>
3383 static LogicalResult verifyExtOp(ExtOp op) {
3384  IntegerType inputType, outputType;
3385  if (isCompatibleVectorType(op.getArg().getType())) {
3386  if (!isCompatibleVectorType(op.getResult().getType()))
3387  return op.emitError(
3388  "input type is a vector but output type is an integer");
3389  if (getVectorNumElements(op.getArg().getType()) !=
3390  getVectorNumElements(op.getResult().getType()))
3391  return op.emitError("input and output vectors are of incompatible shape");
3392  // Because this is a CastOp, the element of vectors is guaranteed to be an
3393  // integer.
3394  inputType = cast<IntegerType>(
3395  cast<VectorType>(op.getArg().getType()).getElementType());
3396  outputType = cast<IntegerType>(
3397  cast<VectorType>(op.getResult().getType()).getElementType());
3398  } else {
3399  // Because this is a CastOp and arg is not a vector, arg is guaranteed to be
3400  // an integer.
3401  inputType = cast<IntegerType>(op.getArg().getType());
3402  outputType = dyn_cast<IntegerType>(op.getResult().getType());
3403  if (!outputType)
3404  return op.emitError(
3405  "input type is an integer but output type is a vector");
3406  }
3407 
3408  if (outputType.getWidth() <= inputType.getWidth())
3409  return op.emitError("integer width of the output type is smaller or "
3410  "equal to the integer width of the input type");
3411  return success();
3412 }
3413 
3414 //===----------------------------------------------------------------------===//
3415 // ZExtOp
3416 //===----------------------------------------------------------------------===//
3417 
3418 LogicalResult ZExtOp::verify() { return verifyExtOp<ZExtOp>(*this); }
3419 
3420 OpFoldResult LLVM::ZExtOp::fold(FoldAdaptor adaptor) {
3421  auto arg = dyn_cast_or_null<IntegerAttr>(adaptor.getArg());
3422  if (!arg)
3423  return {};
3424 
3425  size_t targetSize = cast<IntegerType>(getType()).getWidth();
3426  return IntegerAttr::get(getType(), arg.getValue().zext(targetSize));
3427 }
3428 
3429 //===----------------------------------------------------------------------===//
3430 // SExtOp
3431 //===----------------------------------------------------------------------===//
3432 
3433 LogicalResult SExtOp::verify() { return verifyExtOp<SExtOp>(*this); }
3434 
3435 //===----------------------------------------------------------------------===//
3436 // Folder and verifier for LLVM::BitcastOp
3437 //===----------------------------------------------------------------------===//
3438 
3439 /// Folds a cast op that can be chained.
3440 template <typename T>
3442  typename T::FoldAdaptor adaptor) {
3443  // cast(x : T0, T0) -> x
3444  if (castOp.getArg().getType() == castOp.getType())
3445  return castOp.getArg();
3446  if (auto prev = castOp.getArg().template getDefiningOp<T>()) {
3447  // cast(cast(x : T0, T1), T0) -> x
3448  if (prev.getArg().getType() == castOp.getType())
3449  return prev.getArg();
3450  // cast(cast(x : T0, T1), T2) -> cast(x: T0, T2)
3451  castOp.getArgMutable().set(prev.getArg());
3452  return Value{castOp};
3453  }
3454  return {};
3455 }
3456 
3457 OpFoldResult LLVM::BitcastOp::fold(FoldAdaptor adaptor) {
3458  return foldChainableCast(*this, adaptor);
3459 }
3460 
3461 LogicalResult LLVM::BitcastOp::verify() {
3462  auto resultType = llvm::dyn_cast<LLVMPointerType>(
3463  extractVectorElementType(getResult().getType()));
3464  auto sourceType = llvm::dyn_cast<LLVMPointerType>(
3465  extractVectorElementType(getArg().getType()));
3466 
3467  // If one of the types is a pointer (or vector of pointers), then
3468  // both source and result type have to be pointers.
3469  if (static_cast<bool>(resultType) != static_cast<bool>(sourceType))
3470  return emitOpError("can only cast pointers from and to pointers");
3471 
3472  if (!resultType)
3473  return success();
3474 
3475  auto isVector = llvm::IsaPred<VectorType>;
3476 
3477  // Due to bitcast requiring both operands to be of the same size, it is not
3478  // possible for only one of the two to be a pointer of vectors.
3479  if (isVector(getResult().getType()) && !isVector(getArg().getType()))
3480  return emitOpError("cannot cast pointer to vector of pointers");
3481 
3482  if (!isVector(getResult().getType()) && isVector(getArg().getType()))
3483  return emitOpError("cannot cast vector of pointers to pointer");
3484 
3485  // Bitcast cannot cast between pointers of different address spaces.
3486  // 'llvm.addrspacecast' must be used for this purpose instead.
3487  if (resultType.getAddressSpace() != sourceType.getAddressSpace())
3488  return emitOpError("cannot cast pointers of different address spaces, "
3489  "use 'llvm.addrspacecast' instead");
3490 
3491  return success();
3492 }
3493 
3494 //===----------------------------------------------------------------------===//
3495 // Folder for LLVM::AddrSpaceCastOp
3496 //===----------------------------------------------------------------------===//
3497 
3498 OpFoldResult LLVM::AddrSpaceCastOp::fold(FoldAdaptor adaptor) {
3499  return foldChainableCast(*this, adaptor);
3500 }
3501 
3502 Value LLVM::AddrSpaceCastOp::getViewSource() { return getArg(); }
3503 
3504 //===----------------------------------------------------------------------===//
3505 // Folder for LLVM::GEPOp
3506 //===----------------------------------------------------------------------===//
3507 
3508 OpFoldResult LLVM::GEPOp::fold(FoldAdaptor adaptor) {
3509  GEPIndicesAdaptor<ArrayRef<Attribute>> indices(getRawConstantIndicesAttr(),
3510  adaptor.getDynamicIndices());
3511 
3512  // gep %x:T, 0 -> %x
3513  if (getBase().getType() == getType() && indices.size() == 1)
3514  if (auto integer = llvm::dyn_cast_or_null<IntegerAttr>(indices[0]))
3515  if (integer.getValue().isZero())
3516  return getBase();
3517 
3518  // Canonicalize any dynamic indices of constant value to constant indices.
3519  bool changed = false;
3520  SmallVector<GEPArg> gepArgs;
3521  for (auto iter : llvm::enumerate(indices)) {
3522  auto integer = llvm::dyn_cast_or_null<IntegerAttr>(iter.value());
3523  // Constant indices can only be int32_t, so if integer does not fit we
3524  // are forced to keep it dynamic, despite being a constant.
3525  if (!indices.isDynamicIndex(iter.index()) || !integer ||
3526  !integer.getValue().isSignedIntN(kGEPConstantBitWidth)) {
3527 
3528  PointerUnion<IntegerAttr, Value> existing = getIndices()[iter.index()];
3529  if (Value val = llvm::dyn_cast_if_present<Value>(existing))
3530  gepArgs.emplace_back(val);
3531  else
3532  gepArgs.emplace_back(cast<IntegerAttr>(existing).getInt());
3533 
3534  continue;
3535  }
3536 
3537  changed = true;
3538  gepArgs.emplace_back(integer.getInt());
3539  }
3540  if (changed) {
3541  SmallVector<int32_t> rawConstantIndices;
3542  SmallVector<Value> dynamicIndices;
3543  destructureIndices(getElemType(), gepArgs, rawConstantIndices,
3544  dynamicIndices);
3545 
3546  getDynamicIndicesMutable().assign(dynamicIndices);
3547  setRawConstantIndices(rawConstantIndices);
3548  return Value{*this};
3549  }
3550 
3551  return {};
3552 }
3553 
3554 Value LLVM::GEPOp::getViewSource() { return getBase(); }
3555 
3556 //===----------------------------------------------------------------------===//
3557 // ShlOp
3558 //===----------------------------------------------------------------------===//
3559 
3560 OpFoldResult LLVM::ShlOp::fold(FoldAdaptor adaptor) {
3561  auto rhs = dyn_cast_or_null<IntegerAttr>(adaptor.getRhs());
3562  if (!rhs)
3563  return {};
3564 
3565  if (rhs.getValue().getZExtValue() >=
3566  getLhs().getType().getIntOrFloatBitWidth())
3567  return {}; // TODO: Fold into poison.
3568 
3569  auto lhs = dyn_cast_or_null<IntegerAttr>(adaptor.getLhs());
3570  if (!lhs)
3571  return {};
3572 
3573  return IntegerAttr::get(getType(), lhs.getValue().shl(rhs.getValue()));
3574 }
3575 
3576 //===----------------------------------------------------------------------===//
3577 // OrOp
3578 //===----------------------------------------------------------------------===//
3579 
3580 OpFoldResult LLVM::OrOp::fold(FoldAdaptor adaptor) {
3581  auto lhs = dyn_cast_or_null<IntegerAttr>(adaptor.getLhs());
3582  if (!lhs)
3583  return {};
3584 
3585  auto rhs = dyn_cast_or_null<IntegerAttr>(adaptor.getRhs());
3586  if (!rhs)
3587  return {};
3588 
3589  return IntegerAttr::get(getType(), lhs.getValue() | rhs.getValue());
3590 }
3591 
3592 //===----------------------------------------------------------------------===//
3593 // CallIntrinsicOp
3594 //===----------------------------------------------------------------------===//
3595 
3596 LogicalResult CallIntrinsicOp::verify() {
3597  if (!getIntrin().starts_with("llvm."))
3598  return emitOpError() << "intrinsic name must start with 'llvm.'";
3599  if (failed(verifyOperandBundles(*this)))
3600  return failure();
3601  return success();
3602 }
3603 
3604 void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3605  mlir::StringAttr intrin, mlir::ValueRange args) {
3606  build(builder, state, /*resultTypes=*/TypeRange{}, intrin, args,
3607  FastmathFlagsAttr{},
3608  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
3609  /*res_attrs=*/{});
3610 }
3611 
3612 void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3613  mlir::StringAttr intrin, mlir::ValueRange args,
3614  mlir::LLVM::FastmathFlagsAttr fastMathFlags) {
3615  build(builder, state, /*resultTypes=*/TypeRange{}, intrin, args,
3616  fastMathFlags,
3617  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
3618  /*res_attrs=*/{});
3619 }
3620 
3621 void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3622  mlir::Type resultType, mlir::StringAttr intrin,
3623  mlir::ValueRange args) {
3624  build(builder, state, {resultType}, intrin, args, FastmathFlagsAttr{},
3625  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
3626  /*res_attrs=*/{});
3627 }
3628 
3629 void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3630  mlir::TypeRange resultTypes,
3631  mlir::StringAttr intrin, mlir::ValueRange args,
3632  mlir::LLVM::FastmathFlagsAttr fastMathFlags) {
3633  build(builder, state, resultTypes, intrin, args, fastMathFlags,
3634  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
3635  /*res_attrs=*/{});
3636 }
3637 
3638 ParseResult CallIntrinsicOp::parse(OpAsmParser &parser,
3639  OperationState &result) {
3640  StringAttr intrinAttr;
3643  SmallVector<SmallVector<Type>> opBundleOperandTypes;
3644  ArrayAttr opBundleTags;
3645 
3646  // Parse intrinsic name.
3648  intrinAttr, parser.getBuilder().getType<NoneType>()))
3649  return failure();
3650  result.addAttribute(CallIntrinsicOp::getIntrinAttrName(result.name),
3651  intrinAttr);
3652 
3653  if (parser.parseLParen())
3654  return failure();
3655 
3656  // Parse the function arguments.
3657  if (parser.parseOperandList(operands))
3658  return mlir::failure();
3659 
3660  if (parser.parseRParen())
3661  return mlir::failure();
3662 
3663  // Handle bundles.
3664  SMLoc opBundlesLoc = parser.getCurrentLocation();
3665  if (std::optional<ParseResult> result = parseOpBundles(
3666  parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
3667  result && failed(*result))
3668  return failure();
3669  if (opBundleTags && !opBundleTags.empty())
3670  result.addAttribute(
3671  CallIntrinsicOp::getOpBundleTagsAttrName(result.name).getValue(),
3672  opBundleTags);
3673 
3674  if (parser.parseOptionalAttrDict(result.attributes))
3675  return mlir::failure();
3676 
3677  SmallVector<DictionaryAttr> argAttrs;
3678  SmallVector<DictionaryAttr> resultAttrs;
3679  if (parseCallTypeAndResolveOperands(parser, result, /*isDirect=*/true,
3680  operands, argAttrs, resultAttrs))
3681  return failure();
3683  parser.getBuilder(), result, argAttrs, resultAttrs,
3684  getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
3685 
3686  if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
3687  opBundleOperandTypes,
3688  getOpBundleSizesAttrName(result.name)))
3689  return failure();
3690 
3691  int32_t numOpBundleOperands = 0;
3692  for (const auto &operands : opBundleOperands)
3693  numOpBundleOperands += operands.size();
3694 
3695  result.addAttribute(
3696  CallIntrinsicOp::getOperandSegmentSizeAttr(),
3698  {static_cast<int32_t>(operands.size()), numOpBundleOperands}));
3699 
3700  return mlir::success();
3701 }
3702 
3704  p << ' ';
3705  p.printAttributeWithoutType(getIntrinAttr());
3706 
3707  OperandRange args = getArgs();
3708  p << "(" << args << ")";
3709 
3710  // Operand bundles.
3711  if (!getOpBundleOperands().empty()) {
3712  p << ' ';
3713  printOpBundles(p, *this, getOpBundleOperands(),
3714  getOpBundleOperands().getTypes(), getOpBundleTagsAttr());
3715  }
3716 
3717  p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
3718  {getOperandSegmentSizesAttrName(),
3719  getOpBundleSizesAttrName(), getIntrinAttrName(),
3720  getOpBundleTagsAttrName(), getArgAttrsAttrName(),
3721  getResAttrsAttrName()});
3722 
3723  p << " : ";
3724 
3725  // Reconstruct the MLIR function type from operand and result types.
3727  p, args.getTypes(), getArgAttrsAttr(),
3728  /*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
3729 }
3730 
3731 //===----------------------------------------------------------------------===//
3732 // OpAsmDialectInterface
3733 //===----------------------------------------------------------------------===//
3734 
3735 namespace {
3736 struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface {
3738 
3739  AliasResult getAlias(Attribute attr, raw_ostream &os) const override {
3741  .Case<AccessGroupAttr, AliasScopeAttr, AliasScopeDomainAttr,
3742  DIBasicTypeAttr, DICommonBlockAttr, DICompileUnitAttr,
3743  DICompositeTypeAttr, DIDerivedTypeAttr, DIFileAttr,
3744  DIGlobalVariableAttr, DIGlobalVariableExpressionAttr,
3745  DIImportedEntityAttr, DILabelAttr, DILexicalBlockAttr,
3746  DILexicalBlockFileAttr, DILocalVariableAttr, DIModuleAttr,
3747  DINamespaceAttr, DINullTypeAttr, DIStringTypeAttr,
3748  DISubprogramAttr, DISubroutineTypeAttr, LoopAnnotationAttr,
3749  LoopVectorizeAttr, LoopInterleaveAttr, LoopUnrollAttr,
3750  LoopUnrollAndJamAttr, LoopLICMAttr, LoopDistributeAttr,
3751  LoopPipelineAttr, LoopPeeledAttr, LoopUnswitchAttr, TBAARootAttr,
3752  TBAATagAttr, TBAATypeDescriptorAttr>([&](auto attr) {
3753  os << decltype(attr)::getMnemonic();
3754  return AliasResult::OverridableAlias;
3755  })
3756  .Default([](Attribute) { return AliasResult::NoAlias; });
3757  }
3758 };
3759 } // namespace
3760 
3761 //===----------------------------------------------------------------------===//
3762 // LinkerOptionsOp
3763 //===----------------------------------------------------------------------===//
3764 
3765 LogicalResult LinkerOptionsOp::verify() {
3766  if (mlir::Operation *parentOp = (*this)->getParentOp();
3767  parentOp && !satisfiesLLVMModule(parentOp))
3768  return emitOpError("must appear at the module level");
3769  return success();
3770 }
3771 
3772 //===----------------------------------------------------------------------===//
3773 // ModuleFlagsOp
3774 //===----------------------------------------------------------------------===//
3775 
3776 LogicalResult ModuleFlagsOp::verify() {
3777  if (Operation *parentOp = (*this)->getParentOp();
3778  parentOp && !satisfiesLLVMModule(parentOp))
3779  return emitOpError("must appear at the module level");
3780  for (Attribute flag : getFlags())
3781  if (!isa<ModuleFlagAttr>(flag))
3782  return emitOpError("expected a module flag attribute");
3783  return success();
3784 }
3785 
3786 //===----------------------------------------------------------------------===//
3787 // InlineAsmOp
3788 //===----------------------------------------------------------------------===//
3789 
3790 void InlineAsmOp::getEffects(
3792  &effects) {
3793  if (getHasSideEffects()) {
3794  effects.emplace_back(MemoryEffects::Write::get());
3795  effects.emplace_back(MemoryEffects::Read::get());
3796  }
3797 }
3798 
3799 //===----------------------------------------------------------------------===//
3800 // BlockAddressOp
3801 //===----------------------------------------------------------------------===//
3802 
3803 LogicalResult
3804 BlockAddressOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
3805  Operation *symbol = symbolTable.lookupSymbolIn(parentLLVMModule(*this),
3806  getBlockAddr().getFunction());
3807  auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
3808 
3809  if (!function)
3810  return emitOpError("must reference a function defined by 'llvm.func'");
3811 
3812  return success();
3813 }
3814 
3815 LLVMFuncOp BlockAddressOp::getFunction(SymbolTableCollection &symbolTable) {
3816  return dyn_cast_or_null<LLVMFuncOp>(symbolTable.lookupSymbolIn(
3817  parentLLVMModule(*this), getBlockAddr().getFunction()));
3818 }
3819 
3820 BlockTagOp BlockAddressOp::getBlockTagOp() {
3821  auto funcOp = dyn_cast<LLVMFuncOp>(mlir::SymbolTable::lookupNearestSymbolFrom(
3822  parentLLVMModule(*this), getBlockAddr().getFunction()));
3823  if (!funcOp)
3824  return nullptr;
3825 
3826  BlockTagOp blockTagOp = nullptr;
3827  funcOp.walk([&](LLVM::BlockTagOp labelOp) {
3828  if (labelOp.getTag() == getBlockAddr().getTag()) {
3829  blockTagOp = labelOp;
3830  return WalkResult::interrupt();
3831  }
3832  return WalkResult::advance();
3833  });
3834  return blockTagOp;
3835 }
3836 
3837 LogicalResult BlockAddressOp::verify() {
3838  if (!getBlockTagOp())
3839  return emitOpError(
3840  "expects an existing block label target in the referenced function");
3841 
3842  return success();
3843 }
3844 
3845 /// Fold a blockaddress operation to a dedicated blockaddress
3846 /// attribute.
3847 OpFoldResult BlockAddressOp::fold(FoldAdaptor) { return getBlockAddr(); }
3848 
3849 //===----------------------------------------------------------------------===//
3850 // LLVM::IndirectBrOp
3851 //===----------------------------------------------------------------------===//
3852 
3854  assert(index < getNumSuccessors() && "invalid successor index");
3855  return SuccessorOperands(getSuccOperandsMutable()[index]);
3856 }
3857 
3858 void IndirectBrOp::build(OpBuilder &odsBuilder, OperationState &odsState,
3859  Value addr, ArrayRef<ValueRange> succOperands,
3860  BlockRange successors) {
3861  odsState.addOperands(addr);
3862  for (ValueRange range : succOperands)
3863  odsState.addOperands(range);
3864  SmallVector<int32_t> rangeSegments;
3865  for (ValueRange range : succOperands)
3866  rangeSegments.push_back(range.size());
3867  odsState.getOrAddProperties<Properties>().indbr_operand_segments =
3868  odsBuilder.getDenseI32ArrayAttr(rangeSegments);
3869  odsState.addSuccessors(successors);
3870 }
3871 
3872 static ParseResult parseIndirectBrOpSucessors(
3873  OpAsmParser &parser, Type &flagType,
3874  SmallVectorImpl<Block *> &succOperandBlocks,
3876  SmallVectorImpl<SmallVector<Type>> &succOperandsTypes) {
3877  if (failed(parser.parseCommaSeparatedList(
3879  [&]() {
3880  Block *destination = nullptr;
3881  SmallVector<OpAsmParser::UnresolvedOperand> operands;
3882  SmallVector<Type> operandTypes;
3883 
3884  if (parser.parseSuccessor(destination).failed())
3885  return failure();
3886 
3887  if (succeeded(parser.parseOptionalLParen())) {
3888  if (failed(parser.parseOperandList(
3889  operands, OpAsmParser::Delimiter::None)) ||
3890  failed(parser.parseColonTypeList(operandTypes)) ||
3891  failed(parser.parseRParen()))
3892  return failure();
3893  }
3894  succOperandBlocks.push_back(destination);
3895  succOperands.emplace_back(operands);
3896  succOperandsTypes.emplace_back(operandTypes);
3897  return success();
3898  },
3899  "successor blocks")))
3900  return failure();
3901  return success();
3902 }
3903 
3904 static void
3905 printIndirectBrOpSucessors(OpAsmPrinter &p, IndirectBrOp op, Type flagType,
3906  SuccessorRange succs, OperandRangeRange succOperands,
3907  const TypeRangeRange &succOperandsTypes) {
3908  p << "[";
3909  llvm::interleave(
3910  llvm::zip(succs, succOperands),
3911  [&](auto i) {
3912  p.printNewline();
3913  p.printSuccessorAndUseList(std::get<0>(i), std::get<1>(i));
3914  },
3915  [&] { p << ','; });
3916  if (!succOperands.empty())
3917  p.printNewline();
3918  p << "]";
3919 }
3920 
3921 //===----------------------------------------------------------------------===//
3922 // AssumeOp (intrinsic)
3923 //===----------------------------------------------------------------------===//
3924 
3925 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3926  mlir::Value cond) {
3927  return build(builder, state, cond, /*op_bundle_operands=*/{},
3928  /*op_bundle_tags=*/ArrayAttr{});
3929 }
3930 
3931 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3932  Value cond,
3933  ArrayRef<llvm::OperandBundleDefT<Value>> opBundles) {
3934  SmallVector<ValueRange> opBundleOperands;
3935  SmallVector<Attribute> opBundleTags;
3936  opBundleOperands.reserve(opBundles.size());
3937  opBundleTags.reserve(opBundles.size());
3938 
3939  for (const llvm::OperandBundleDefT<Value> &bundle : opBundles) {
3940  opBundleOperands.emplace_back(bundle.inputs());
3941  opBundleTags.push_back(
3942  StringAttr::get(builder.getContext(), bundle.getTag()));
3943  }
3944 
3945  auto opBundleTagsAttr = ArrayAttr::get(builder.getContext(), opBundleTags);
3946  return build(builder, state, cond, opBundleOperands, opBundleTagsAttr);
3947 }
3948 
3949 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3950  Value cond, llvm::StringRef tag, ValueRange args) {
3951  llvm::OperandBundleDefT<Value> opBundle(
3952  tag.str(), SmallVector<Value>(args.begin(), args.end()));
3953  return build(builder, state, cond, opBundle);
3954 }
3955 
3956 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3957  Value cond, AssumeAlignTag, Value ptr, Value align) {
3958  return build(builder, state, cond, "align", ValueRange{ptr, align});
3959 }
3960 
3961 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3962  Value cond, AssumeSeparateStorageTag, Value ptr1,
3963  Value ptr2) {
3964  return build(builder, state, cond, "separate_storage",
3965  ValueRange{ptr1, ptr2});
3966 }
3967 
3968 LogicalResult LLVM::AssumeOp::verify() { return verifyOperandBundles(*this); }
3969 
3970 //===----------------------------------------------------------------------===//
3971 // masked_gather (intrinsic)
3972 //===----------------------------------------------------------------------===//
3973 
3974 LogicalResult LLVM::masked_gather::verify() {
3975  auto ptrsVectorType = getPtrs().getType();
3976  Type expectedPtrsVectorType =
3978  LLVM::getVectorNumElements(getRes().getType()));
3979  // Vector of pointers type should match result vector type, other than the
3980  // element type.
3981  if (ptrsVectorType != expectedPtrsVectorType)
3982  return emitOpError("expected operand #1 type to be ")
3983  << expectedPtrsVectorType;
3984  return success();
3985 }
3986 
3987 //===----------------------------------------------------------------------===//
3988 // masked_scatter (intrinsic)
3989 //===----------------------------------------------------------------------===//
3990 
3991 LogicalResult LLVM::masked_scatter::verify() {
3992  auto ptrsVectorType = getPtrs().getType();
3993  Type expectedPtrsVectorType =
3995  LLVM::getVectorNumElements(getValue().getType()));
3996  // Vector of pointers type should match value vector type, other than the
3997  // element type.
3998  if (ptrsVectorType != expectedPtrsVectorType)
3999  return emitOpError("expected operand #2 type to be ")
4000  << expectedPtrsVectorType;
4001  return success();
4002 }
4003 
4004 //===----------------------------------------------------------------------===//
4005 // LLVMDialect initialization, type parsing, and registration.
4006 //===----------------------------------------------------------------------===//
4007 
4008 void LLVMDialect::initialize() {
4009  registerAttributes();
4010 
4011  // clang-format off
4012  addTypes<LLVMVoidType,
4013  LLVMTokenType,
4014  LLVMLabelType,
4015  LLVMMetadataType>();
4016  // clang-format on
4017  registerTypes();
4018 
4019  addOperations<
4020 #define GET_OP_LIST
4021 #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
4022  ,
4023 #define GET_OP_LIST
4024 #include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
4025  >();
4026 
4027  // Support unknown operations because not all LLVM operations are registered.
4028  allowUnknownOperations();
4029  // clang-format off
4030  addInterfaces<LLVMOpAsmDialectInterface>();
4031  // clang-format on
4032  declarePromisedInterface<DialectInlinerInterface, LLVMDialect>();
4033 }
4034 
4035 #define GET_OP_CLASSES
4036 #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
4037 
4038 #define GET_OP_CLASSES
4039 #include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
4040 
4041 LogicalResult LLVMDialect::verifyDataLayoutString(
4042  StringRef descr, llvm::function_ref<void(const Twine &)> reportError) {
4043  llvm::Expected<llvm::DataLayout> maybeDataLayout =
4044  llvm::DataLayout::parse(descr);
4045  if (maybeDataLayout)
4046  return success();
4047 
4048  std::string message;
4049  llvm::raw_string_ostream messageStream(message);
4050  llvm::logAllUnhandledErrors(maybeDataLayout.takeError(), messageStream);
4051  reportError("invalid data layout descriptor: " + message);
4052  return failure();
4053 }
4054 
4055 /// Verify LLVM dialect attributes.
4056 LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op,
4057  NamedAttribute attr) {
4058  // If the data layout attribute is present, it must use the LLVM data layout
4059  // syntax. Try parsing it and report errors in case of failure. Users of this
4060  // attribute may assume it is well-formed and can pass it to the (asserting)
4061  // llvm::DataLayout constructor.
4062  if (attr.getName() != LLVM::LLVMDialect::getDataLayoutAttrName())
4063  return success();
4064  if (auto stringAttr = llvm::dyn_cast<StringAttr>(attr.getValue()))
4065  return verifyDataLayoutString(
4066  stringAttr.getValue(),
4067  [op](const Twine &message) { op->emitOpError() << message.str(); });
4068 
4069  return op->emitOpError() << "expected '"
4070  << LLVM::LLVMDialect::getDataLayoutAttrName()
4071  << "' to be a string attributes";
4072 }
4073 
4074 LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op,
4075  Type paramType,
4076  NamedAttribute paramAttr) {
4077  // LLVM attribute may be attached to a result of operation that has not been
4078  // converted to LLVM dialect yet, so the result may have a type with unknown
4079  // representation in LLVM dialect type space. In this case we cannot verify
4080  // whether the attribute may be
4081  bool verifyValueType = isCompatibleType(paramType);
4082  StringAttr name = paramAttr.getName();
4083 
4084  auto checkUnitAttrType = [&]() -> LogicalResult {
4085  if (!llvm::isa<UnitAttr>(paramAttr.getValue()))
4086  return op->emitError() << name << " should be a unit attribute";
4087  return success();
4088  };
4089  auto checkTypeAttrType = [&]() -> LogicalResult {
4090  if (!llvm::isa<TypeAttr>(paramAttr.getValue()))
4091  return op->emitError() << name << " should be a type attribute";
4092  return success();
4093  };
4094  auto checkIntegerAttrType = [&]() -> LogicalResult {
4095  if (!llvm::isa<IntegerAttr>(paramAttr.getValue()))
4096  return op->emitError() << name << " should be an integer attribute";
4097  return success();
4098  };
4099  auto checkPointerType = [&]() -> LogicalResult {
4100  if (!llvm::isa<LLVMPointerType>(paramType))
4101  return op->emitError()
4102  << name << " attribute attached to non-pointer LLVM type";
4103  return success();
4104  };
4105  auto checkIntegerType = [&]() -> LogicalResult {
4106  if (!llvm::isa<IntegerType>(paramType))
4107  return op->emitError()
4108  << name << " attribute attached to non-integer LLVM type";
4109  return success();
4110  };
4111  auto checkPointerTypeMatches = [&]() -> LogicalResult {
4112  if (failed(checkPointerType()))
4113  return failure();
4114 
4115  return success();
4116  };
4117 
4118  // Check a unit attribute that is attached to a pointer value.
4119  if (name == LLVMDialect::getNoAliasAttrName() ||
4120  name == LLVMDialect::getReadonlyAttrName() ||
4121  name == LLVMDialect::getReadnoneAttrName() ||
4122  name == LLVMDialect::getWriteOnlyAttrName() ||
4123  name == LLVMDialect::getNestAttrName() ||
4124  name == LLVMDialect::getNoCaptureAttrName() ||
4125  name == LLVMDialect::getNoFreeAttrName() ||
4126  name == LLVMDialect::getNonNullAttrName()) {
4127  if (failed(checkUnitAttrType()))
4128  return failure();
4129  if (verifyValueType && failed(checkPointerType()))
4130  return failure();
4131  return success();
4132  }
4133 
4134  // Check a type attribute that is attached to a pointer value.
4135  if (name == LLVMDialect::getStructRetAttrName() ||
4136  name == LLVMDialect::getByValAttrName() ||
4137  name == LLVMDialect::getByRefAttrName() ||
4138  name == LLVMDialect::getElementTypeAttrName() ||
4139  name == LLVMDialect::getInAllocaAttrName() ||
4140  name == LLVMDialect::getPreallocatedAttrName()) {
4141  if (failed(checkTypeAttrType()))
4142  return failure();
4143  if (verifyValueType && failed(checkPointerTypeMatches()))
4144  return failure();
4145  return success();
4146  }
4147 
4148  // Check a unit attribute that is attached to an integer value.
4149  if (name == LLVMDialect::getSExtAttrName() ||
4150  name == LLVMDialect::getZExtAttrName()) {
4151  if (failed(checkUnitAttrType()))
4152  return failure();
4153  if (verifyValueType && failed(checkIntegerType()))
4154  return failure();
4155  return success();
4156  }
4157 
4158  // Check an integer attribute that is attached to a pointer value.
4159  if (name == LLVMDialect::getAlignAttrName() ||
4160  name == LLVMDialect::getDereferenceableAttrName() ||
4161  name == LLVMDialect::getDereferenceableOrNullAttrName()) {
4162  if (failed(checkIntegerAttrType()))
4163  return failure();
4164  if (verifyValueType && failed(checkPointerType()))
4165  return failure();
4166  return success();
4167  }
4168 
4169  // Check an integer attribute that is attached to a pointer value.
4170  if (name == LLVMDialect::getStackAlignmentAttrName()) {
4171  if (failed(checkIntegerAttrType()))
4172  return failure();
4173  return success();
4174  }
4175 
4176  // Check a unit attribute that can be attached to arbitrary types.
4177  if (name == LLVMDialect::getNoUndefAttrName() ||
4178  name == LLVMDialect::getInRegAttrName() ||
4179  name == LLVMDialect::getReturnedAttrName())
4180  return checkUnitAttrType();
4181 
4182  return success();
4183 }
4184 
4185 /// Verify LLVMIR function argument attributes.
4186 LogicalResult LLVMDialect::verifyRegionArgAttribute(Operation *op,
4187  unsigned regionIdx,
4188  unsigned argIdx,
4189  NamedAttribute argAttr) {
4190  auto funcOp = dyn_cast<FunctionOpInterface>(op);
4191  if (!funcOp)
4192  return success();
4193  Type argType = funcOp.getArgumentTypes()[argIdx];
4194 
4195  return verifyParameterAttribute(op, argType, argAttr);
4196 }
4197 
4198 LogicalResult LLVMDialect::verifyRegionResultAttribute(Operation *op,
4199  unsigned regionIdx,
4200  unsigned resIdx,
4201  NamedAttribute resAttr) {
4202  auto funcOp = dyn_cast<FunctionOpInterface>(op);
4203  if (!funcOp)
4204  return success();
4205  Type resType = funcOp.getResultTypes()[resIdx];
4206 
4207  // Check to see if this function has a void return with a result attribute
4208  // to it. It isn't clear what semantics we would assign to that.
4209  if (llvm::isa<LLVMVoidType>(resType))
4210  return op->emitError() << "cannot attach result attributes to functions "
4211  "with a void return";
4212 
4213  // Check to see if this attribute is allowed as a result attribute. Only
4214  // explicitly forbidden LLVM attributes will cause an error.
4215  auto name = resAttr.getName();
4216  if (name == LLVMDialect::getAllocAlignAttrName() ||
4217  name == LLVMDialect::getAllocatedPointerAttrName() ||
4218  name == LLVMDialect::getByValAttrName() ||
4219  name == LLVMDialect::getByRefAttrName() ||
4220  name == LLVMDialect::getInAllocaAttrName() ||
4221  name == LLVMDialect::getNestAttrName() ||
4222  name == LLVMDialect::getNoCaptureAttrName() ||
4223  name == LLVMDialect::getNoFreeAttrName() ||
4224  name == LLVMDialect::getPreallocatedAttrName() ||
4225  name == LLVMDialect::getReadnoneAttrName() ||
4226  name == LLVMDialect::getReadonlyAttrName() ||
4227  name == LLVMDialect::getReturnedAttrName() ||
4228  name == LLVMDialect::getStackAlignmentAttrName() ||
4229  name == LLVMDialect::getStructRetAttrName() ||
4230  name == LLVMDialect::getWriteOnlyAttrName())
4231  return op->emitError() << name << " is not a valid result attribute";
4232  return verifyParameterAttribute(op, resType, resAttr);
4233 }
4234 
4236  Type type, Location loc) {
4237  // If this was folded from an operation other than llvm.mlir.constant, it
4238  // should be materialized as such. Note that an llvm.mlir.zero may fold into
4239  // a builtin zero attribute and thus will materialize as a llvm.mlir.constant.
4240  if (auto symbol = dyn_cast<FlatSymbolRefAttr>(value))
4241  if (isa<LLVM::LLVMPointerType>(type))
4242  return builder.create<LLVM::AddressOfOp>(loc, type, symbol);
4243  if (isa<LLVM::UndefAttr>(value))
4244  return builder.create<LLVM::UndefOp>(loc, type);
4245  if (isa<LLVM::PoisonAttr>(value))
4246  return builder.create<LLVM::PoisonOp>(loc, type);
4247  if (isa<LLVM::ZeroAttr>(value))
4248  return builder.create<LLVM::ZeroOp>(loc, type);
4249  // Otherwise try materializing it as a regular llvm.mlir.constant op.
4250  return LLVM::ConstantOp::materialize(builder, value, type, loc);
4251 }
4252 
4253 //===----------------------------------------------------------------------===//
4254 // Utility functions.
4255 //===----------------------------------------------------------------------===//
4256 
4258  StringRef name, StringRef value,
4259  LLVM::Linkage linkage) {
4260  assert(builder.getInsertionBlock() &&
4261  builder.getInsertionBlock()->getParentOp() &&
4262  "expected builder to point to a block constrained in an op");
4263  auto module =
4264  builder.getInsertionBlock()->getParentOp()->getParentOfType<ModuleOp>();
4265  assert(module && "builder points to an op outside of a module");
4266 
4267  // Create the global at the entry of the module.
4268  OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
4269  MLIRContext *ctx = builder.getContext();
4270  auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
4271  auto global = moduleBuilder.create<LLVM::GlobalOp>(
4272  loc, type, /*isConstant=*/true, linkage, name,
4273  builder.getStringAttr(value), /*alignment=*/0);
4274 
4275  LLVMPointerType ptrType = LLVMPointerType::get(ctx);
4276  // Get the pointer to the first character in the global string.
4277  Value globalPtr =
4278  builder.create<LLVM::AddressOfOp>(loc, ptrType, global.getSymNameAttr());
4279  return builder.create<LLVM::GEPOp>(loc, ptrType, type, globalPtr,
4280  ArrayRef<GEPArg>{0, 0});
4281 }
4282 
4284  return op->hasTrait<OpTrait::SymbolTable>() &&
4286 }
static OperandRange getSuccessorOperands(Block *block, unsigned successorIndex)
Return the operand range used to transfer operands from block to its successor with the given index.
Definition: CFGToSCF.cpp:142
static Value getBase(Value v)
Looks through known "view-like" ops to find the base memref.
static Operation * materializeConstant(Dialect *dialect, OpBuilder &builder, Attribute value, Type type, Location loc)
A utility function used to materialize a constant for a given attribute and type.
Definition: FoldUtils.cpp:50
static MLIRContext * getContext(OpFoldResult val)
static int parseOptionalKeywordAlternative(OpAsmParser &parser, ArrayRef< StringRef > keywords)
Definition: LLVMDialect.cpp:99
LogicalResult verifyCallOpVarCalleeType(OpTy callOp)
Verify that the parameter and return types of the variadic callee type match the callOp argument and ...
LogicalResult checkGlobalXtorData(Operation *op, ArrayAttr data)
static ParseResult parseGEPIndices(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &indices, DenseI32ArrayAttr &rawConstantIndices)
static LogicalResult verifyOperandBundles(OpType &op)
static ParseResult parseCmpOp(OpAsmParser &parser, OperationState &result)
LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType, ArrayRef< AtomicOrdering > unsupportedOrderings)
Verifies the attributes and the type of atomic memory access operations.
static void printOneOpBundle(OpAsmPrinter &p, OperandRange operands, TypeRange operandTypes, StringRef tag)
static LogicalResult verifyComdat(Operation *op, std::optional< SymbolRefAttr > attr)
static RetTy parseOptionalLLVMKeyword(OpAsmParser &parser, OperationState &result, EnumTy defaultValue)
Parse an enum from the keyword, or default to the provided default value.
static LLVMFunctionType getLLVMFuncType(MLIRContext *context, TypeRange results, ValueRange args)
Constructs a LLVMFunctionType from MLIR results and args.
static void printSwitchOpCases(OpAsmPrinter &p, SwitchOp op, Type flagType, DenseIntElementsAttr caseValues, SuccessorRange caseDestinations, OperandRangeRange caseOperands, const TypeRangeRange &caseOperandTypes)
static ParseResult resolveOpBundleOperands(OpAsmParser &parser, SMLoc loc, OperationState &state, ArrayRef< SmallVector< OpAsmParser::UnresolvedOperand >> opBundleOperands, ArrayRef< SmallVector< Type >> opBundleOperandTypes, StringAttr opBundleSizesAttrName)
static ParseResult parseOptionalCallFuncPtr(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &operands)
Parses an optional function pointer operand before the call argument list for indirect calls,...
static bool isZeroAttribute(Attribute value)
static void printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp, OperandRange indices, DenseI32ArrayAttr rawConstantIndices)
static LLVMStructType getValAndBoolStructType(Type valType)
Returns an LLVM struct type that contains a value type and a boolean type.
static void printOpBundles(OpAsmPrinter &p, Operation *op, OperandRangeRange opBundleOperands, TypeRangeRange opBundleOperandTypes, std::optional< ArrayAttr > opBundleTags)
static void printShuffleType(AsmPrinter &printer, Operation *op, Type v1Type, Type resType, DenseI32ArrayAttr mask)
Nothing to do when the result type is inferred.
static LogicalResult verifyBlockTags(LLVMFuncOp funcOp)
static Type buildLLVMFunctionType(OpAsmParser &parser, SMLoc loc, ArrayRef< Type > inputs, ArrayRef< Type > outputs, function_interface_impl::VariadicFlag variadicFlag)
static auto processFMFAttr(ArrayRef< NamedAttribute > attrs)
Definition: LLVMDialect.cpp:58
static std::optional< ParseResult > parseOpBundles(OpAsmParser &p, SmallVector< SmallVector< OpAsmParser::UnresolvedOperand >> &opBundleOperands, SmallVector< SmallVector< Type >> &opBundleOperandTypes, ArrayAttr &opBundleTags)
static Operation * parentLLVMModule(Operation *op)
static ParseResult parseSwitchOpCases(OpAsmParser &parser, Type flagType, DenseIntElementsAttr &caseValues, SmallVectorImpl< Block * > &caseDestinations, SmallVectorImpl< SmallVector< OpAsmParser::UnresolvedOperand >> &caseOperands, SmallVectorImpl< SmallVector< Type >> &caseOperandTypes)
<cases> ::= [ (case (, case )* )? ] <case> ::= integer : bb-id (( ssa-use-and-type-list ))?
static TypeAttr getCallOpVarCalleeType(LLVMFunctionType calleeType)
Gets the variadic callee type for a LLVMFunctionType.
static Type getInsertExtractValueElementType(function_ref< InFlightDiagnostic(StringRef)> emitError, Type containerType, ArrayRef< int64_t > position)
Extract the type at position in the LLVM IR aggregate type containerType.
static ParseResult parseOneOpBundle(OpAsmParser &p, SmallVector< SmallVector< OpAsmParser::UnresolvedOperand >> &opBundleOperands, SmallVector< SmallVector< Type >> &opBundleOperandTypes, SmallVector< Attribute > &opBundleTags)
static ParseResult parseIndirectBrOpSucessors(OpAsmParser &parser, Type &flagType, SmallVectorImpl< Block * > &succOperandBlocks, SmallVectorImpl< SmallVector< OpAsmParser::UnresolvedOperand >> &succOperands, SmallVectorImpl< SmallVector< Type >> &succOperandsTypes)
static void printIndirectBrOpSucessors(OpAsmPrinter &p, IndirectBrOp op, Type flagType, SuccessorRange succs, OperandRangeRange succOperands, const TypeRangeRange &succOperandsTypes)
static ParseResult parseCallTypeAndResolveOperands(OpAsmParser &parser, OperationState &result, bool isDirect, ArrayRef< OpAsmParser::UnresolvedOperand > operands, SmallVectorImpl< DictionaryAttr > &argAttrs, SmallVectorImpl< DictionaryAttr > &resultAttrs)
Parses the type of a call operation and resolves the operands if the parsing succeeds.
static LogicalResult verifySymbolAttrUse(FlatSymbolRefAttr symbol, Operation *op, SymbolTableCollection &symbolTable)
Verifies symbol's use in op to ensure the symbol is a valid and fully defined llvm....
Definition: LLVMDialect.cpp:73
static Type extractVectorElementType(Type type)
Returns the elemental type of any LLVM-compatible vector type or self.
static bool hasScalableVectorType(Type t)
Check if the given type is a scalable vector type or a vector/array type that contains a nested scala...
static OpFoldResult foldChainableCast(T castOp, typename T::FoldAdaptor adaptor)
Folds a cast op that can be chained.
static void destructureIndices(Type currType, ArrayRef< GEPArg > indices, SmallVectorImpl< int32_t > &rawConstantIndices, SmallVectorImpl< Value > &dynamicIndices)
Destructures the 'indices' parameter into 'rawConstantIndices' and 'dynamicIndices',...
static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser, OperationState &result)
Parse common attributes that might show up in the same order in both GlobalOp and AliasOp.
static SmallVector< Type, 1 > getCallOpResultTypes(LLVMFunctionType calleeType)
Gets the MLIR Op-like result types of a LLVMFunctionType.
static Type getI1SameShape(Type type)
Returns a boolean type that has the same shape as type.
Definition: LLVMDialect.cpp:89
static Attribute getBoolAttribute(Type type, MLIRContext *ctx, bool value)
Returns a scalar or vector boolean attribute of the given type.
static LogicalResult verifyCallOpDebugInfo(CallOp callOp, LLVMFuncOp callee)
Verify that an inlinable callsite of a debug-info-bearing function in a debug-info-bearing function h...
static ParseResult parseShuffleType(AsmParser &parser, Type v1Type, Type &resType, DenseI32ArrayAttr mask)
Build the result type of a shuffle vector operation.
static LogicalResult verifyExtOp(ExtOp op)
Verifies that the given extension operation operates on consistent scalars or vectors,...
static constexpr const char kElemTypeAttrName[]
Definition: LLVMDialect.cpp:56
static ParseResult parseInsertExtractValueElementType(AsmParser &parser, Type &valueType, Type containerType, DenseI64ArrayAttr position)
Infer the value type from the container type and position.
static LogicalResult verifyStructIndices(Type baseGEPType, unsigned indexPos, GEPIndicesAdaptor< ValueRange > indices, function_ref< InFlightDiagnostic()> emitOpError)
For the given indices, check if they comply with baseGEPType, especially check against LLVMStructType...
static int64_t getNumElements(Type t)
Compute the total number of elements in the given type, also taking into account nested types.
static bool isTypeCompatibleWithAtomicOp(Type type, const DataLayout &dataLayout)
Returns true if the given type is supported by atomic operations.
static void printInsertExtractValueElementType(AsmPrinter &printer, Operation *op, Type valueType, Type containerType, DenseI64ArrayAttr position)
Nothing to print for an inferred type.
#define REGISTER_ENUM_TYPE(Ty)
@ None
static std::string diag(const llvm::Value &value)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static void print(spirv::VerCapExtAttr triple, DialectAsmPrinter &printer)
static Type getElementType(Type type, ArrayRef< int32_t > indices, function_ref< InFlightDiagnostic(StringRef)> emitErrorFn)
Walks the given type hierarchy with the given indices, potentially down to component granularity,...
Definition: SPIRVOps.cpp:188
static VectorType getVectorType(Type scalarTy, const VectorizationStrategy *strategy)
Returns the vector type resulting from applying the provided vectorization strategy on the scalar typ...
The possible results of an alias query.
Definition: AliasAnalysis.h:26
@ NoAlias
The two locations do not alias at all.
Definition: AliasAnalysis.h:34
This base class exposes generic asm parser hooks, usable across the various derived parsers.
ParseResult parseSymbolName(StringAttr &result)
Parse an -identifier and store it (without the '@' symbol) in a string attribute.
@ Paren
Parens surrounding zero or more operands.
@ None
Zero or more operands with no delimiters.
@ Square
Square brackets surrounding zero or more operands.
virtual OptionalParseResult parseOptionalInteger(APInt &result)=0
Parse an optional integer value from the stream.
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
Definition: AsmPrinter.cpp:73
virtual ParseResult parseRParen()=0
Parse a ) token.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
virtual ParseResult parseLSquare()=0
Parse a [ token.
virtual ParseResult parseRSquare()=0
Parse a ] token.
virtual ParseResult parseOptionalColonTypeList(SmallVectorImpl< Type > &result)=0
Parse an optional colon followed by a type list, which if present must have at least one type.
ParseResult parseInteger(IntT &result)
Parse an integer value from the stream.
virtual ParseResult parseOptionalRParen()=0
Parse a ) token if present.
virtual ParseResult parseCustomAttributeWithFallback(Attribute &result, Type type, function_ref< ParseResult(Attribute &result, Type type)> parseAttribute)=0
Parse a custom attribute with the provided callback, unless the next token is #, in which case the ge...
ParseResult parseString(std::string *string)
Parse a quoted string token.
virtual ParseResult parseOptionalAttrDictWithKeyword(NamedAttrList &result)=0
Parse a named dictionary into 'result' if the attributes keyword is present.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual ParseResult parseOptionalComma()=0
Parse a , token if present.
virtual ParseResult parseColon()=0
Parse a : token.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseOptionalRSquare()=0
Parse a ] token if present.
virtual ParseResult parseLParen()=0
Parse a ( token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
virtual ParseResult parseComma()=0
Parse a , token.
virtual ParseResult parseOptionalLParen()=0
Parse a ( token if present.
ParseResult parseTypeList(SmallVectorImpl< Type > &result)
Parse a type list.
Definition: AsmPrinter.cpp:78
ParseResult parseKeyword(StringRef keyword)
Parse a given keyword.
virtual ParseResult parseOptionalLSquare()=0
Parse a [ token if present.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
This base class exposes generic asm printer hooks, usable across the various derived printers.
virtual void printAttributeWithoutType(Attribute attr)
Print the given attribute without its type.
virtual void printSymbolName(StringRef symbolRef)
Print the given string as a symbol reference, i.e.
virtual void printString(StringRef string)
Print the given string as a quoted string, escaping any special or non-printable characters in it.
virtual void printAttribute(Attribute attr)
Attributes are known-constant values of operations.
Definition: Attributes.h:25
This class provides an abstraction over the different types of ranges over Blocks.
Definition: BlockSupport.h:106
Block represents an ordered list of Operations.
Definition: Block.h:33
bool empty()
Definition: Block.h:148
BlockArgument getArgument(unsigned i)
Definition: Block.h:129
Operation * getTerminator()
Get the terminator operation of this block.
Definition: Block.cpp:246
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
Definition: Block.cpp:155
Operation & front()
Definition: Block.h:153
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
Definition: Block.cpp:33
static BoolAttr get(MLIRContext *context, bool value)
This class is a general helper class for creating context-global objects like types,...
Definition: Builders.h:51
UnitAttr getUnitAttr()
Definition: Builders.cpp:94
IntegerAttr getI32IntegerAttr(int32_t value)
Definition: Builders.cpp:196
DenseI32ArrayAttr getDenseI32ArrayAttr(ArrayRef< int32_t > values)
Definition: Builders.cpp:159
IntegerAttr getI64IntegerAttr(int64_t value)
Definition: Builders.cpp:108
Ty getType(Args &&...args)
Get or construct an instance of the type Ty with provided arguments.
Definition: Builders.h:89
StringAttr getStringAttr(const Twine &bytes)
Definition: Builders.cpp:258
TypedAttr getZeroAttr(Type type)
Definition: Builders.cpp:320
MLIRContext * getContext() const
Definition: Builders.h:56
Attr getAttr(Args &&...args)
Get or construct an instance of the attribute Attr with provided arguments.
Definition: Builders.h:96
The main mechanism for performing data layout queries.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
An attribute that represents a reference to a dense vector or tensor object.
std::enable_if_t<!std::is_base_of< Attribute, T >::value||std::is_same< Attribute, T >::value, T > getSplatValue() const
Return the splat value for this attribute.
auto getValues() const
Return the held element values as a range of the given type.
Type getElementType() const
Return the element type of this DenseElementsAttr.
static DenseElementsAttr get(ShapedType type, ArrayRef< Attribute > values)
Constructs a dense elements attribute from an array of element values.
An attribute that represents a reference to a dense integer vector or tensor object.
static DenseIntElementsAttr get(const ShapedType &type, Arg &&arg)
Get an instance of a DenseIntElementsAttr with the given arguments.
A symbol reference with a reference path containing a single element.
StringRef getValue() const
Returns the name of the held symbol reference.
StringAttr getAttr() const
Returns the name of the held symbol reference as a StringAttr.
This class represents a fused location whose metadata is known to be an instance of the given type.
Definition: Location.h:139
This class represents a diagnostic that is inflight and set to be reported.
Definition: Diagnostics.h:314
Diagnostic & attachNote(std::optional< Location > noteLoc=std::nullopt)
Attaches a note to this diagnostic.
Definition: Diagnostics.h:352
Class used for building a 'llvm.getelementptr'.
Definition: LLVMDialect.h:76
Class used for convenient access and iteration over GEP indices.
Definition: LLVMDialect.h:123
bool isDynamicIndex(size_t index) const
Returns whether the GEP index at the given position is a dynamic index.
Definition: LLVMDialect.h:152
size_t size() const
Returns the amount of indices of the GEPOp.
Definition: LLVMDialect.h:157
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:66
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:60
This class provides a mutable adaptor for a range of operands.
Definition: ValueRange.h:118
std::optional< NamedAttribute > getNamed(StringRef name) const
Return the specified named attribute if present, std::nullopt otherwise.
Attribute erase(StringAttr name)
Erase the attribute with the given name from the list.
void append(StringRef name, Attribute attr)
Add an attribute with the specified name.
Attribute set(StringAttr name, Attribute value)
If the an attribute exists with the specified name, change it to the new value.
NamedAttribute represents a combination of a name and an Attribute value.
Definition: Attributes.h:164
StringAttr getName() const
Return the name of the attribute.
Definition: Attributes.cpp:55
Attribute getValue() const
Return the value of the attribute.
Definition: Attributes.h:179
OpAsmDialectInterface(Dialect *dialect)
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult parseRegion(Region &region, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region.
virtual ParseResult parseSuccessor(Block *&dest)=0
Parse a single operation successor.
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
virtual OptionalParseResult parseOptionalOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single operand if present.
virtual ParseResult parseSuccessorAndUseList(Block *&dest, SmallVectorImpl< Value > &operands)=0
Parse a single operation successor and its operand list.
virtual OptionalParseResult parseOptionalRegion(Region &region, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region if present.
ParseResult resolveOperands(Operands &&operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
virtual ParseResult parseOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None, bool allowResultNumber=true, int requiredOperandCount=-1)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printNewline()=0
Print a newline and indent the printer to the start of the current operation.
virtual void printSuccessorAndUseList(Block *successor, ValueRange succOperands)=0
Print the successor and its operands.
void printOperands(const ContainerType &container)
Print a comma separated list of operands.
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:346
This class helps build Operations.
Definition: Builders.h:205
Listener * getListener() const
Returns the current listener of this builder, or nullptr if this builder doesn't have a listener.
Definition: Builders.h:318
Block * createBlock(Region *parent, Region::iterator insertPt={}, TypeRange argTypes=std::nullopt, ArrayRef< Location > locs=std::nullopt)
Add new block with 'argTypes' arguments and set the insertion point to the end of it.
Definition: Builders.cpp:426
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:453
Block * getInsertionBlock() const
Return the block the current insertion point belongs to.
Definition: Builders.h:440
This class represents a single result from folding an operation.
Definition: OpDefinition.h:271
This class provides the API for ops that are known to be isolated from above.
A trait used to provide symbol table functionalities to a region operation.
Definition: SymbolTable.h:435
This class represents a contiguous range of operand ranges, e.g.
Definition: ValueRange.h:84
This class implements the operand iterators for the Operation class.
Definition: ValueRange.h:43
type_range getTypes() const
Definition: ValueRange.cpp:28
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Definition: Operation.h:750
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition: Operation.h:234
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:268
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
Definition: Operation.h:238
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:671
This class implements Optional functionality for ParseResult.
Definition: OpDefinition.h:39
ParseResult value() const
Access the internal ParseResult value.
Definition: OpDefinition.h:52
bool has_value() const
Returns true if we contain a valid ParseResult value.
Definition: OpDefinition.h:49
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
iterator_range< OpIterator > getOps()
Definition: Region.h:172
bool empty()
Definition: Region.h:60
Block & emplaceBlock()
Definition: Region.h:46
This class represents a specific instance of an effect.
static DerivedEffect * get()
Returns a unique instance for the derived effect class.
This class models how operands are forwarded to block arguments in control flow.
This class implements the successor iterators for Block.
Definition: BlockSupport.h:73
This class represents a collection of SymbolTables.
Definition: SymbolTable.h:283
Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
Operation * lookupSymbolIn(Operation *symbolTableOp, StringAttr symbol)
Look up a symbol with the specified name within the specified symbol table operation,...
static StringRef getSymbolAttrName()
Return the name of the attribute used for symbol names.
Definition: SymbolTable.h:76
static Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
This class provides an abstraction for a range of TypeRange.
Definition: TypeRange.h:95
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:37
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
Definition: Types.cpp:35
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition: Types.cpp:122
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:387
type_range getTypes() const
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Type getType() const
Return the type of this value.
Definition: Value.h:105
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:20
A utility result that is used to signal how to proceed with an ongoing walk:
Definition: Visitors.h:33
static WalkResult skip()
Definition: Visitors.h:52
static WalkResult advance()
Definition: Visitors.h:51
bool wasInterrupted() const
Returns true if the walk was interrupted.
Definition: Visitors.h:55
static WalkResult interrupt()
Definition: Visitors.h:50
static DenseArrayAttrImpl get(MLIRContext *context, ArrayRef< int32_t > content)
Builder from ArrayRef<T>.
A named class for passing around the variadic flag.
Value createGlobalString(Location loc, OpBuilder &builder, StringRef name, StringRef value, Linkage linkage)
Create an LLVM global containing the string "value" at the module containing surrounding the insertio...
Type getVectorType(Type elementType, unsigned numElements, bool isScalable=false)
Creates an LLVM dialect-compatible vector type with the given element type and length.
Definition: LLVMTypes.cpp:838
bool isScalableVectorType(Type vectorType)
Returns whether a vector type is scalable or not.
Definition: LLVMTypes.cpp:832
bool isCompatibleVectorType(Type type)
Returns true if the given type is a vector type compatible with the LLVM dialect.
Definition: LLVMTypes.cpp:811
bool isCompatibleOuterType(Type type)
Returns true if the given outer type is compatible with the LLVM dialect without checking its potenti...
Definition: LLVMTypes.cpp:702
bool satisfiesLLVMModule(Operation *op)
LLVM requires some operations to be inside of a Module operation.
constexpr int kGEPConstantBitWidth
Bit-width of a 'GEPConstantIndex' within GEPArg.
Definition: LLVMDialect.h:67
bool isCompatibleType(Type type)
Returns true if the given type is compatible with the LLVM dialect.
Definition: LLVMTypes.cpp:793
bool isCompatibleFloatingPointType(Type type)
Returns true if the given type is a floating-point type compatible with the LLVM dialect.
Definition: LLVMTypes.cpp:806
llvm::ElementCount getVectorNumElements(Type type)
Returns the element count of any LLVM-compatible vector type.
Definition: LLVMTypes.cpp:824
void printFunctionSignature(OpAsmPrinter &p, TypeRange argTypes, ArrayAttr argAttrs, bool isVariadic, TypeRange resultTypes, ArrayAttr resultAttrs, Region *body=nullptr, bool printEmptyResult=true)
Print a function signature for a call or callable operation.
ParseResult parseFunctionSignature(OpAsmParser &parser, SmallVectorImpl< Type > &argTypes, SmallVectorImpl< DictionaryAttr > &argAttrs, SmallVectorImpl< Type > &resultTypes, SmallVectorImpl< DictionaryAttr > &resultAttrs, bool mustParseEmptyResult=true)
Parses a function signature using parser.
void addArgAndResultAttrs(Builder &builder, OperationState &result, ArrayRef< DictionaryAttr > argAttrs, ArrayRef< DictionaryAttr > resultAttrs, StringAttr argAttrsName, StringAttr resAttrsName)
Adds argument and result attributes, provided as argAttrs and resultAttrs arguments,...
void walk(Operation *op, function_ref< void(Region *)> callback, WalkOrder order)
Walk all of the regions, blocks, or operations nested under (and including) the given operation.
Definition: Visitors.h:136
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:344
ParseResult parseFunctionSignatureWithArguments(OpAsmParser &parser, bool allowVariadic, SmallVectorImpl< OpAsmParser::Argument > &arguments, bool &isVariadic, SmallVectorImpl< Type > &resultTypes, SmallVectorImpl< DictionaryAttr > &resultAttrs)
Parses a function signature using parser.
void printFunctionAttributes(OpAsmPrinter &p, Operation *op, ArrayRef< StringRef > elided={})
Prints the list of function prefixed with the "attributes" keyword.
void printFunctionSignature(OpAsmPrinter &p, FunctionOpInterface op, ArrayRef< Type > argTypes, bool isVariadic, ArrayRef< Type > resultTypes)
Prints the signature of the function-like operation op.
Operation::operand_range getIndices(Operation *op)
Get the indices that the given load/store operation is operating on.
Definition: Utils.cpp:18
QueryRef parse(llvm::StringRef line, const QuerySession &qs)
Definition: Query.cpp:20
Value constantIndex(OpBuilder &builder, Location loc, int64_t i)
Generates a constant of index type.
Definition: CodegenUtils.h:331
Visibility
This enum describes C++ inheritance visibility.
Definition: Class.h:408
std::string stringify(T &&t)
Generically convert a value to a std::string.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:490
detail::constant_int_value_binder m_ConstantInt(IntegerAttr::ValueType *bind_value)
Matches a constant holding a scalar/vector/tensor integer (splat) and writes the integer value to bin...
Definition: Matchers.h:527
const FrozenRewritePatternSet GreedyRewriteConfig bool * changed
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
Definition: Utils.cpp:305
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
detail::constant_op_matcher m_Constant()
Matches a constant foldable operation.
Definition: Matchers.h:369
LogicalResult verify(Operation *op, bool verifyRecursively=true)
Perform (potentially expensive) checks of invariants, used to detect compiler bugs,...
Definition: Verifier.cpp:424
A callable is either a symbol, or an SSA value, that is referenced by a call-like operation.
This is the representation of an operand reference.
This represents an operation in an abstracted form, suitable for use with the builder APIs.
T & getOrAddProperties()
Get (or create) a properties of the provided type to be set on the operation on creation.
SmallVector< Value, 4 > operands
void addOperands(ValueRange newOperands)
void addAttributes(ArrayRef< NamedAttribute > newAttributes)
Add an array of named attributes.
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
void addSuccessors(Block *successor)
Adds a successor to the operation sate. successor must not be null.
void addTypes(ArrayRef< Type > newTypes)
NamedAttrList attributes
Region * addRegion()
Create a region that should be attached to the operation.