MLIR  20.0.0git
LLVMDialect.cpp
Go to the documentation of this file.
1 //===- LLVMDialect.cpp - LLVM IR Ops and Dialect registration -------------===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 //
9 // This file defines the types and operation details for the LLVM IR dialect in
10 // MLIR, and the LLVM IR dialect. It also registers the dialect.
11 //
12 //===----------------------------------------------------------------------===//
13 
15 #include "TypeDetail.h"
19 #include "mlir/IR/Attributes.h"
20 #include "mlir/IR/Builders.h"
21 #include "mlir/IR/BuiltinOps.h"
22 #include "mlir/IR/BuiltinTypes.h"
24 #include "mlir/IR/MLIRContext.h"
25 #include "mlir/IR/Matchers.h"
28 
29 #include "llvm/ADT/SCCIterator.h"
30 #include "llvm/ADT/TypeSwitch.h"
31 #include "llvm/AsmParser/Parser.h"
32 #include "llvm/Bitcode/BitcodeReader.h"
33 #include "llvm/Bitcode/BitcodeWriter.h"
34 #include "llvm/IR/Attributes.h"
35 #include "llvm/IR/Function.h"
36 #include "llvm/IR/Type.h"
37 #include "llvm/Support/Error.h"
38 #include "llvm/Support/Mutex.h"
39 #include "llvm/Support/SourceMgr.h"
40 
41 #include <numeric>
42 #include <optional>
43 
44 using namespace mlir;
45 using namespace mlir::LLVM;
46 using mlir::LLVM::cconv::getMaxEnumValForCConv;
47 using mlir::LLVM::linkage::getMaxEnumValForLinkage;
48 using mlir::LLVM::tailcallkind::getMaxEnumValForTailCallKind;
49 
50 #include "mlir/Dialect/LLVMIR/LLVMOpsDialect.cpp.inc"
51 
52 //===----------------------------------------------------------------------===//
53 // Property Helpers
54 //===----------------------------------------------------------------------===//
55 
56 //===----------------------------------------------------------------------===//
57 // IntegerOverflowFlags
58 
59 namespace mlir {
61  IntegerOverflowFlags flags) {
62  return IntegerOverflowFlagsAttr::get(ctx, flags);
63 }
64 
65 static LogicalResult
66 convertFromAttribute(IntegerOverflowFlags &flags, Attribute attr,
68  auto flagsAttr = dyn_cast<IntegerOverflowFlagsAttr>(attr);
69  if (!flagsAttr) {
70  return emitError() << "expected 'overflowFlags' attribute to be an "
71  "IntegerOverflowFlagsAttr, but got "
72  << attr;
73  }
74  flags = flagsAttr.getValue();
75  return success();
76 }
77 } // namespace mlir
78 
79 static ParseResult parseOverflowFlags(AsmParser &p,
80  IntegerOverflowFlags &flags) {
81  if (failed(p.parseOptionalKeyword("overflow"))) {
82  flags = IntegerOverflowFlags::none;
83  return success();
84  }
85  if (p.parseLess())
86  return failure();
87  do {
88  StringRef kw;
89  SMLoc loc = p.getCurrentLocation();
90  if (p.parseKeyword(&kw))
91  return failure();
92  std::optional<IntegerOverflowFlags> flag =
93  symbolizeIntegerOverflowFlags(kw);
94  if (!flag)
95  return p.emitError(loc,
96  "invalid overflow flag: expected nsw, nuw, or none");
97  flags = flags | *flag;
98  } while (succeeded(p.parseOptionalComma()));
99  return p.parseGreater();
100 }
101 
103  IntegerOverflowFlags flags) {
104  if (flags == IntegerOverflowFlags::none)
105  return;
106  p << " overflow<";
108  if (bitEnumContainsAny(flags, IntegerOverflowFlags::nsw))
109  strs.push_back("nsw");
110  if (bitEnumContainsAny(flags, IntegerOverflowFlags::nuw))
111  strs.push_back("nuw");
112  llvm::interleaveComma(strs, p);
113  p << ">";
114 }
115 
116 //===----------------------------------------------------------------------===//
117 // Attribute Helpers
118 //===----------------------------------------------------------------------===//
119 
120 static constexpr const char kElemTypeAttrName[] = "elem_type";
121 
123  SmallVector<NamedAttribute, 8> filteredAttrs(
124  llvm::make_filter_range(attrs, [&](NamedAttribute attr) {
125  if (attr.getName() == "fastmathFlags") {
126  auto defAttr =
127  FastmathFlagsAttr::get(attr.getValue().getContext(), {});
128  return defAttr != attr.getValue();
129  }
130  return true;
131  }));
132  return filteredAttrs;
133 }
134 
135 /// Verifies `symbol`'s use in `op` to ensure the symbol is a valid and
136 /// fully defined llvm.func.
137 static LogicalResult verifySymbolAttrUse(FlatSymbolRefAttr symbol,
138  Operation *op,
139  SymbolTableCollection &symbolTable) {
140  StringRef name = symbol.getValue();
141  auto func =
142  symbolTable.lookupNearestSymbolFrom<LLVMFuncOp>(op, symbol.getAttr());
143  if (!func)
144  return op->emitOpError("'")
145  << name << "' does not reference a valid LLVM function";
146  if (func.isExternal())
147  return op->emitOpError("'") << name << "' does not have a definition";
148  return success();
149 }
150 
151 /// Returns a boolean type that has the same shape as `type`. It supports both
152 /// fixed size vectors as well as scalable vectors.
153 static Type getI1SameShape(Type type) {
154  Type i1Type = IntegerType::get(type.getContext(), 1);
156  return LLVM::getVectorType(i1Type, LLVM::getVectorNumElements(type));
157  return i1Type;
158 }
159 
160 // Parses one of the keywords provided in the list `keywords` and returns the
161 // position of the parsed keyword in the list. If none of the keywords from the
162 // list is parsed, returns -1.
164  ArrayRef<StringRef> keywords) {
165  for (const auto &en : llvm::enumerate(keywords)) {
166  if (succeeded(parser.parseOptionalKeyword(en.value())))
167  return en.index();
168  }
169  return -1;
170 }
171 
172 namespace {
173 template <typename Ty>
174 struct EnumTraits {};
175 
176 #define REGISTER_ENUM_TYPE(Ty) \
177  template <> \
178  struct EnumTraits<Ty> { \
179  static StringRef stringify(Ty value) { return stringify##Ty(value); } \
180  static unsigned getMaxEnumVal() { return getMaxEnumValFor##Ty(); } \
181  }
182 
183 REGISTER_ENUM_TYPE(Linkage);
184 REGISTER_ENUM_TYPE(UnnamedAddr);
185 REGISTER_ENUM_TYPE(CConv);
186 REGISTER_ENUM_TYPE(TailCallKind);
188 } // namespace
189 
190 /// Parse an enum from the keyword, or default to the provided default value.
191 /// The return type is the enum type by default, unless overridden with the
192 /// second template argument.
193 template <typename EnumTy, typename RetTy = EnumTy>
195  OperationState &result,
196  EnumTy defaultValue) {
198  for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
199  names.push_back(EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i)));
200 
201  int index = parseOptionalKeywordAlternative(parser, names);
202  if (index == -1)
203  return static_cast<RetTy>(defaultValue);
204  return static_cast<RetTy>(index);
205 }
206 
207 //===----------------------------------------------------------------------===//
208 // Operand bundle helpers.
209 //===----------------------------------------------------------------------===//
210 
211 static void printOneOpBundle(OpAsmPrinter &p, OperandRange operands,
212  TypeRange operandTypes, StringRef tag) {
213  p.printString(tag);
214  p << "(";
215 
216  if (!operands.empty()) {
217  p.printOperands(operands);
218  p << " : ";
219  llvm::interleaveComma(operandTypes, p);
220  }
221 
222  p << ")";
223 }
224 
226  OperandRangeRange opBundleOperands,
227  TypeRangeRange opBundleOperandTypes,
228  std::optional<ArrayAttr> opBundleTags) {
229  if (opBundleOperands.empty())
230  return;
231  assert(opBundleTags && "expect operand bundle tags");
232 
233  p << "[";
234  llvm::interleaveComma(
235  llvm::zip(opBundleOperands, opBundleOperandTypes, *opBundleTags), p,
236  [&p](auto bundle) {
237  auto bundleTag = cast<StringAttr>(std::get<2>(bundle)).getValue();
238  printOneOpBundle(p, std::get<0>(bundle), std::get<1>(bundle),
239  bundleTag);
240  });
241  p << "]";
242 }
243 
244 static ParseResult parseOneOpBundle(
245  OpAsmParser &p,
247  SmallVector<SmallVector<Type>> &opBundleOperandTypes,
248  SmallVector<Attribute> &opBundleTags) {
249  SMLoc currentParserLoc = p.getCurrentLocation();
251  SmallVector<Type> types;
252  std::string tag;
253 
254  if (p.parseString(&tag))
255  return p.emitError(currentParserLoc, "expect operand bundle tag");
256 
257  if (p.parseLParen())
258  return failure();
259 
260  if (p.parseOptionalRParen()) {
261  if (p.parseOperandList(operands) || p.parseColon() ||
262  p.parseTypeList(types) || p.parseRParen())
263  return failure();
264  }
265 
266  opBundleOperands.push_back(std::move(operands));
267  opBundleOperandTypes.push_back(std::move(types));
268  opBundleTags.push_back(StringAttr::get(p.getContext(), tag));
269 
270  return success();
271 }
272 
273 static std::optional<ParseResult> parseOpBundles(
274  OpAsmParser &p,
276  SmallVector<SmallVector<Type>> &opBundleOperandTypes,
277  ArrayAttr &opBundleTags) {
278  if (p.parseOptionalLSquare())
279  return std::nullopt;
280 
281  if (succeeded(p.parseOptionalRSquare()))
282  return success();
283 
284  SmallVector<Attribute> opBundleTagAttrs;
285  auto bundleParser = [&] {
286  return parseOneOpBundle(p, opBundleOperands, opBundleOperandTypes,
287  opBundleTagAttrs);
288  };
289  if (p.parseCommaSeparatedList(bundleParser))
290  return failure();
291 
292  if (p.parseRSquare())
293  return failure();
294 
295  opBundleTags = ArrayAttr::get(p.getContext(), opBundleTagAttrs);
296 
297  return success();
298 }
299 
300 //===----------------------------------------------------------------------===//
301 // Printing, parsing, folding and builder for LLVM::CmpOp.
302 //===----------------------------------------------------------------------===//
303 
304 void ICmpOp::print(OpAsmPrinter &p) {
305  p << " \"" << stringifyICmpPredicate(getPredicate()) << "\" " << getOperand(0)
306  << ", " << getOperand(1);
307  p.printOptionalAttrDict((*this)->getAttrs(), {"predicate"});
308  p << " : " << getLhs().getType();
309 }
310 
311 void FCmpOp::print(OpAsmPrinter &p) {
312  p << " \"" << stringifyFCmpPredicate(getPredicate()) << "\" " << getOperand(0)
313  << ", " << getOperand(1);
314  p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()), {"predicate"});
315  p << " : " << getLhs().getType();
316 }
317 
318 // <operation> ::= `llvm.icmp` string-literal ssa-use `,` ssa-use
319 // attribute-dict? `:` type
320 // <operation> ::= `llvm.fcmp` string-literal ssa-use `,` ssa-use
321 // attribute-dict? `:` type
322 template <typename CmpPredicateType>
323 static ParseResult parseCmpOp(OpAsmParser &parser, OperationState &result) {
324  StringAttr predicateAttr;
326  Type type;
327  SMLoc predicateLoc, trailingTypeLoc;
328  if (parser.getCurrentLocation(&predicateLoc) ||
329  parser.parseAttribute(predicateAttr, "predicate", result.attributes) ||
330  parser.parseOperand(lhs) || parser.parseComma() ||
331  parser.parseOperand(rhs) ||
332  parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
333  parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type) ||
334  parser.resolveOperand(lhs, type, result.operands) ||
335  parser.resolveOperand(rhs, type, result.operands))
336  return failure();
337 
338  // Replace the string attribute `predicate` with an integer attribute.
339  int64_t predicateValue = 0;
340  if (std::is_same<CmpPredicateType, ICmpPredicate>()) {
341  std::optional<ICmpPredicate> predicate =
342  symbolizeICmpPredicate(predicateAttr.getValue());
343  if (!predicate)
344  return parser.emitError(predicateLoc)
345  << "'" << predicateAttr.getValue()
346  << "' is an incorrect value of the 'predicate' attribute";
347  predicateValue = static_cast<int64_t>(*predicate);
348  } else {
349  std::optional<FCmpPredicate> predicate =
350  symbolizeFCmpPredicate(predicateAttr.getValue());
351  if (!predicate)
352  return parser.emitError(predicateLoc)
353  << "'" << predicateAttr.getValue()
354  << "' is an incorrect value of the 'predicate' attribute";
355  predicateValue = static_cast<int64_t>(*predicate);
356  }
357 
358  result.attributes.set("predicate",
359  parser.getBuilder().getI64IntegerAttr(predicateValue));
360 
361  // The result type is either i1 or a vector type <? x i1> if the inputs are
362  // vectors.
363  if (!isCompatibleType(type))
364  return parser.emitError(trailingTypeLoc,
365  "expected LLVM dialect-compatible type");
366  result.addTypes(getI1SameShape(type));
367  return success();
368 }
369 
370 ParseResult ICmpOp::parse(OpAsmParser &parser, OperationState &result) {
371  return parseCmpOp<ICmpPredicate>(parser, result);
372 }
373 
374 ParseResult FCmpOp::parse(OpAsmParser &parser, OperationState &result) {
375  return parseCmpOp<FCmpPredicate>(parser, result);
376 }
377 
378 /// Returns a scalar or vector boolean attribute of the given type.
379 static Attribute getBoolAttribute(Type type, MLIRContext *ctx, bool value) {
380  auto boolAttr = BoolAttr::get(ctx, value);
381  ShapedType shapedType = dyn_cast<ShapedType>(type);
382  if (!shapedType)
383  return boolAttr;
384  return DenseElementsAttr::get(shapedType, boolAttr);
385 }
386 
387 OpFoldResult ICmpOp::fold(FoldAdaptor adaptor) {
388  if (getPredicate() != ICmpPredicate::eq &&
389  getPredicate() != ICmpPredicate::ne)
390  return {};
391 
392  // cmpi(eq/ne, x, x) -> true/false
393  if (getLhs() == getRhs())
394  return getBoolAttribute(getType(), getContext(),
395  getPredicate() == ICmpPredicate::eq);
396 
397  // cmpi(eq/ne, alloca, null) -> false/true
398  if (getLhs().getDefiningOp<AllocaOp>() && getRhs().getDefiningOp<ZeroOp>())
399  return getBoolAttribute(getType(), getContext(),
400  getPredicate() == ICmpPredicate::ne);
401 
402  // cmpi(eq/ne, null, alloca) -> cmpi(eq/ne, alloca, null)
403  if (getLhs().getDefiningOp<ZeroOp>() && getRhs().getDefiningOp<AllocaOp>()) {
404  Value lhs = getLhs();
405  Value rhs = getRhs();
406  getLhsMutable().assign(rhs);
407  getRhsMutable().assign(lhs);
408  return getResult();
409  }
410 
411  return {};
412 }
413 
414 //===----------------------------------------------------------------------===//
415 // Printing, parsing and verification for LLVM::AllocaOp.
416 //===----------------------------------------------------------------------===//
417 
418 void AllocaOp::print(OpAsmPrinter &p) {
419  auto funcTy =
420  FunctionType::get(getContext(), {getArraySize().getType()}, {getType()});
421 
422  if (getInalloca())
423  p << " inalloca";
424 
425  p << ' ' << getArraySize() << " x " << getElemType();
426  if (getAlignment() && *getAlignment() != 0)
427  p.printOptionalAttrDict((*this)->getAttrs(),
428  {kElemTypeAttrName, getInallocaAttrName()});
429  else
431  (*this)->getAttrs(),
432  {getAlignmentAttrName(), kElemTypeAttrName, getInallocaAttrName()});
433  p << " : " << funcTy;
434 }
435 
436 // <operation> ::= `llvm.alloca` `inalloca`? ssa-use `x` type
437 // attribute-dict? `:` type `,` type
438 ParseResult AllocaOp::parse(OpAsmParser &parser, OperationState &result) {
440  Type type, elemType;
441  SMLoc trailingTypeLoc;
442 
443  if (succeeded(parser.parseOptionalKeyword("inalloca")))
444  result.addAttribute(getInallocaAttrName(result.name),
445  UnitAttr::get(parser.getContext()));
446 
447  if (parser.parseOperand(arraySize) || parser.parseKeyword("x") ||
448  parser.parseType(elemType) ||
449  parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
450  parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type))
451  return failure();
452 
453  std::optional<NamedAttribute> alignmentAttr =
454  result.attributes.getNamed("alignment");
455  if (alignmentAttr.has_value()) {
456  auto alignmentInt = llvm::dyn_cast<IntegerAttr>(alignmentAttr->getValue());
457  if (!alignmentInt)
458  return parser.emitError(parser.getNameLoc(),
459  "expected integer alignment");
460  if (alignmentInt.getValue().isZero())
461  result.attributes.erase("alignment");
462  }
463 
464  // Extract the result type from the trailing function type.
465  auto funcType = llvm::dyn_cast<FunctionType>(type);
466  if (!funcType || funcType.getNumInputs() != 1 ||
467  funcType.getNumResults() != 1)
468  return parser.emitError(
469  trailingTypeLoc,
470  "expected trailing function type with one argument and one result");
471 
472  if (parser.resolveOperand(arraySize, funcType.getInput(0), result.operands))
473  return failure();
474 
475  Type resultType = funcType.getResult(0);
476  if (auto ptrResultType = llvm::dyn_cast<LLVMPointerType>(resultType))
477  result.addAttribute(kElemTypeAttrName, TypeAttr::get(elemType));
478 
479  result.addTypes({funcType.getResult(0)});
480  return success();
481 }
482 
483 LogicalResult AllocaOp::verify() {
484  // Only certain target extension types can be used in 'alloca'.
485  if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getElemType());
486  targetExtType && !targetExtType.supportsMemOps())
487  return emitOpError()
488  << "this target extension type cannot be used in alloca";
489 
490  return success();
491 }
492 
493 //===----------------------------------------------------------------------===//
494 // LLVM::BrOp
495 //===----------------------------------------------------------------------===//
496 
498  assert(index == 0 && "invalid successor index");
499  return SuccessorOperands(getDestOperandsMutable());
500 }
501 
502 //===----------------------------------------------------------------------===//
503 // LLVM::CondBrOp
504 //===----------------------------------------------------------------------===//
505 
507  assert(index < getNumSuccessors() && "invalid successor index");
508  return SuccessorOperands(index == 0 ? getTrueDestOperandsMutable()
509  : getFalseDestOperandsMutable());
510 }
511 
512 void CondBrOp::build(OpBuilder &builder, OperationState &result,
513  Value condition, Block *trueDest, ValueRange trueOperands,
514  Block *falseDest, ValueRange falseOperands,
515  std::optional<std::pair<uint32_t, uint32_t>> weights) {
516  DenseI32ArrayAttr weightsAttr;
517  if (weights)
518  weightsAttr =
519  builder.getDenseI32ArrayAttr({static_cast<int32_t>(weights->first),
520  static_cast<int32_t>(weights->second)});
521 
522  build(builder, result, condition, trueOperands, falseOperands, weightsAttr,
523  /*loop_annotation=*/{}, trueDest, falseDest);
524 }
525 
526 //===----------------------------------------------------------------------===//
527 // LLVM::SwitchOp
528 //===----------------------------------------------------------------------===//
529 
530 void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
531  Block *defaultDestination, ValueRange defaultOperands,
532  DenseIntElementsAttr caseValues,
533  BlockRange caseDestinations,
534  ArrayRef<ValueRange> caseOperands,
535  ArrayRef<int32_t> branchWeights) {
536  DenseI32ArrayAttr weightsAttr;
537  if (!branchWeights.empty())
538  weightsAttr = builder.getDenseI32ArrayAttr(branchWeights);
539 
540  build(builder, result, value, defaultOperands, caseOperands, caseValues,
541  weightsAttr, defaultDestination, caseDestinations);
542 }
543 
544 void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
545  Block *defaultDestination, ValueRange defaultOperands,
546  ArrayRef<APInt> caseValues, BlockRange caseDestinations,
547  ArrayRef<ValueRange> caseOperands,
548  ArrayRef<int32_t> branchWeights) {
549  DenseIntElementsAttr caseValuesAttr;
550  if (!caseValues.empty()) {
551  ShapedType caseValueType = VectorType::get(
552  static_cast<int64_t>(caseValues.size()), value.getType());
553  caseValuesAttr = DenseIntElementsAttr::get(caseValueType, caseValues);
554  }
555 
556  build(builder, result, value, defaultDestination, defaultOperands,
557  caseValuesAttr, caseDestinations, caseOperands, branchWeights);
558 }
559 
560 void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
561  Block *defaultDestination, ValueRange defaultOperands,
562  ArrayRef<int32_t> caseValues, BlockRange caseDestinations,
563  ArrayRef<ValueRange> caseOperands,
564  ArrayRef<int32_t> branchWeights) {
565  DenseIntElementsAttr caseValuesAttr;
566  if (!caseValues.empty()) {
567  ShapedType caseValueType = VectorType::get(
568  static_cast<int64_t>(caseValues.size()), value.getType());
569  caseValuesAttr = DenseIntElementsAttr::get(caseValueType, caseValues);
570  }
571 
572  build(builder, result, value, defaultDestination, defaultOperands,
573  caseValuesAttr, caseDestinations, caseOperands, branchWeights);
574 }
575 
576 /// <cases> ::= `[` (case (`,` case )* )? `]`
577 /// <case> ::= integer `:` bb-id (`(` ssa-use-and-type-list `)`)?
578 static ParseResult parseSwitchOpCases(
579  OpAsmParser &parser, Type flagType, DenseIntElementsAttr &caseValues,
580  SmallVectorImpl<Block *> &caseDestinations,
582  SmallVectorImpl<SmallVector<Type>> &caseOperandTypes) {
583  if (failed(parser.parseLSquare()))
584  return failure();
585  if (succeeded(parser.parseOptionalRSquare()))
586  return success();
587  SmallVector<APInt> values;
588  unsigned bitWidth = flagType.getIntOrFloatBitWidth();
589  auto parseCase = [&]() {
590  int64_t value = 0;
591  if (failed(parser.parseInteger(value)))
592  return failure();
593  values.push_back(APInt(bitWidth, value, /*isSigned=*/true));
594 
595  Block *destination;
597  SmallVector<Type> operandTypes;
598  if (parser.parseColon() || parser.parseSuccessor(destination))
599  return failure();
600  if (!parser.parseOptionalLParen()) {
601  if (parser.parseOperandList(operands, OpAsmParser::Delimiter::None,
602  /*allowResultNumber=*/false) ||
603  parser.parseColonTypeList(operandTypes) || parser.parseRParen())
604  return failure();
605  }
606  caseDestinations.push_back(destination);
607  caseOperands.emplace_back(operands);
608  caseOperandTypes.emplace_back(operandTypes);
609  return success();
610  };
611  if (failed(parser.parseCommaSeparatedList(parseCase)))
612  return failure();
613 
614  ShapedType caseValueType =
615  VectorType::get(static_cast<int64_t>(values.size()), flagType);
616  caseValues = DenseIntElementsAttr::get(caseValueType, values);
617  return parser.parseRSquare();
618 }
619 
620 static void printSwitchOpCases(OpAsmPrinter &p, SwitchOp op, Type flagType,
621  DenseIntElementsAttr caseValues,
622  SuccessorRange caseDestinations,
623  OperandRangeRange caseOperands,
624  const TypeRangeRange &caseOperandTypes) {
625  p << '[';
626  p.printNewline();
627  if (!caseValues) {
628  p << ']';
629  return;
630  }
631 
632  size_t index = 0;
633  llvm::interleave(
634  llvm::zip(caseValues, caseDestinations),
635  [&](auto i) {
636  p << " ";
637  p << std::get<0>(i).getLimitedValue();
638  p << ": ";
639  p.printSuccessorAndUseList(std::get<1>(i), caseOperands[index++]);
640  },
641  [&] {
642  p << ',';
643  p.printNewline();
644  });
645  p.printNewline();
646  p << ']';
647 }
648 
649 LogicalResult SwitchOp::verify() {
650  if ((!getCaseValues() && !getCaseDestinations().empty()) ||
651  (getCaseValues() &&
652  getCaseValues()->size() !=
653  static_cast<int64_t>(getCaseDestinations().size())))
654  return emitOpError("expects number of case values to match number of "
655  "case destinations");
656  if (getBranchWeights() && getBranchWeights()->size() != getNumSuccessors())
657  return emitError("expects number of branch weights to match number of "
658  "successors: ")
659  << getBranchWeights()->size() << " vs " << getNumSuccessors();
660  if (getCaseValues() &&
661  getValue().getType() != getCaseValues()->getElementType())
662  return emitError("expects case value type to match condition value type");
663  return success();
664 }
665 
667  assert(index < getNumSuccessors() && "invalid successor index");
668  return SuccessorOperands(index == 0 ? getDefaultOperandsMutable()
669  : getCaseOperandsMutable(index - 1));
670 }
671 
672 //===----------------------------------------------------------------------===//
673 // Code for LLVM::GEPOp.
674 //===----------------------------------------------------------------------===//
675 
676 constexpr int32_t GEPOp::kDynamicIndex;
677 
679  return GEPIndicesAdaptor<ValueRange>(getRawConstantIndicesAttr(),
680  getDynamicIndices());
681 }
682 
683 /// Returns the elemental type of any LLVM-compatible vector type or self.
685  if (auto vectorType = llvm::dyn_cast<VectorType>(type))
686  return vectorType.getElementType();
687  if (auto scalableVectorType = llvm::dyn_cast<LLVMScalableVectorType>(type))
688  return scalableVectorType.getElementType();
689  if (auto fixedVectorType = llvm::dyn_cast<LLVMFixedVectorType>(type))
690  return fixedVectorType.getElementType();
691  return type;
692 }
693 
694 /// Destructures the 'indices' parameter into 'rawConstantIndices' and
695 /// 'dynamicIndices', encoding the former in the process. In the process,
696 /// dynamic indices which are used to index into a structure type are converted
697 /// to constant indices when possible. To do this, the GEPs element type should
698 /// be passed as first parameter.
699 static void destructureIndices(Type currType, ArrayRef<GEPArg> indices,
700  SmallVectorImpl<int32_t> &rawConstantIndices,
701  SmallVectorImpl<Value> &dynamicIndices) {
702  for (const GEPArg &iter : indices) {
703  // If the thing we are currently indexing into is a struct we must turn
704  // any integer constants into constant indices. If this is not possible
705  // we don't do anything here. The verifier will catch it and emit a proper
706  // error. All other canonicalization is done in the fold method.
707  bool requiresConst = !rawConstantIndices.empty() &&
708  isa_and_nonnull<LLVMStructType>(currType);
709  if (Value val = llvm::dyn_cast_if_present<Value>(iter)) {
710  APInt intC;
711  if (requiresConst && matchPattern(val, m_ConstantInt(&intC)) &&
712  intC.isSignedIntN(kGEPConstantBitWidth)) {
713  rawConstantIndices.push_back(intC.getSExtValue());
714  } else {
715  rawConstantIndices.push_back(GEPOp::kDynamicIndex);
716  dynamicIndices.push_back(val);
717  }
718  } else {
719  rawConstantIndices.push_back(iter.get<GEPConstantIndex>());
720  }
721 
722  // Skip for very first iteration of this loop. First index does not index
723  // within the aggregates, but is just a pointer offset.
724  if (rawConstantIndices.size() == 1 || !currType)
725  continue;
726 
727  currType =
728  TypeSwitch<Type, Type>(currType)
729  .Case<VectorType, LLVMScalableVectorType, LLVMFixedVectorType,
730  LLVMArrayType>([](auto containerType) {
731  return containerType.getElementType();
732  })
733  .Case([&](LLVMStructType structType) -> Type {
734  int64_t memberIndex = rawConstantIndices.back();
735  if (memberIndex >= 0 && static_cast<size_t>(memberIndex) <
736  structType.getBody().size())
737  return structType.getBody()[memberIndex];
738  return nullptr;
739  })
740  .Default(Type(nullptr));
741  }
742 }
743 
744 void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
745  Type elementType, Value basePtr, ArrayRef<GEPArg> indices,
746  bool inbounds, ArrayRef<NamedAttribute> attributes) {
747  SmallVector<int32_t> rawConstantIndices;
748  SmallVector<Value> dynamicIndices;
749  destructureIndices(elementType, indices, rawConstantIndices, dynamicIndices);
750 
751  result.addTypes(resultType);
752  result.addAttributes(attributes);
753  result.addAttribute(getRawConstantIndicesAttrName(result.name),
754  builder.getDenseI32ArrayAttr(rawConstantIndices));
755  if (inbounds) {
756  result.addAttribute(getInboundsAttrName(result.name),
757  builder.getUnitAttr());
758  }
759  result.addAttribute(kElemTypeAttrName, TypeAttr::get(elementType));
760  result.addOperands(basePtr);
761  result.addOperands(dynamicIndices);
762 }
763 
764 void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
765  Type elementType, Value basePtr, ValueRange indices,
766  bool inbounds, ArrayRef<NamedAttribute> attributes) {
767  build(builder, result, resultType, elementType, basePtr,
768  SmallVector<GEPArg>(indices), inbounds, attributes);
769 }
770 
771 static ParseResult
774  DenseI32ArrayAttr &rawConstantIndices) {
775  SmallVector<int32_t> constantIndices;
776 
777  auto idxParser = [&]() -> ParseResult {
778  int32_t constantIndex;
779  OptionalParseResult parsedInteger =
781  if (parsedInteger.has_value()) {
782  if (failed(parsedInteger.value()))
783  return failure();
784  constantIndices.push_back(constantIndex);
785  return success();
786  }
787 
788  constantIndices.push_back(LLVM::GEPOp::kDynamicIndex);
789  return parser.parseOperand(indices.emplace_back());
790  };
791  if (parser.parseCommaSeparatedList(idxParser))
792  return failure();
793 
794  rawConstantIndices =
795  DenseI32ArrayAttr::get(parser.getContext(), constantIndices);
796  return success();
797 }
798 
799 static void printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp,
800  OperandRange indices,
801  DenseI32ArrayAttr rawConstantIndices) {
802  llvm::interleaveComma(
803  GEPIndicesAdaptor<OperandRange>(rawConstantIndices, indices), printer,
805  if (Value val = llvm::dyn_cast_if_present<Value>(cst))
806  printer.printOperand(val);
807  else
808  printer << cst.get<IntegerAttr>().getInt();
809  });
810 }
811 
812 /// For the given `indices`, check if they comply with `baseGEPType`,
813 /// especially check against LLVMStructTypes nested within.
814 static LogicalResult
815 verifyStructIndices(Type baseGEPType, unsigned indexPos,
817  function_ref<InFlightDiagnostic()> emitOpError) {
818  if (indexPos >= indices.size())
819  // Stop searching
820  return success();
821 
822  return TypeSwitch<Type, LogicalResult>(baseGEPType)
823  .Case<LLVMStructType>([&](LLVMStructType structType) -> LogicalResult {
824  if (!indices[indexPos].is<IntegerAttr>())
825  return emitOpError() << "expected index " << indexPos
826  << " indexing a struct to be constant";
827 
828  int32_t gepIndex = indices[indexPos].get<IntegerAttr>().getInt();
829  ArrayRef<Type> elementTypes = structType.getBody();
830  if (gepIndex < 0 ||
831  static_cast<size_t>(gepIndex) >= elementTypes.size())
832  return emitOpError() << "index " << indexPos
833  << " indexing a struct is out of bounds";
834 
835  // Instead of recursively going into every children types, we only
836  // dive into the one indexed by gepIndex.
837  return verifyStructIndices(elementTypes[gepIndex], indexPos + 1,
838  indices, emitOpError);
839  })
840  .Case<VectorType, LLVMScalableVectorType, LLVMFixedVectorType,
841  LLVMArrayType>([&](auto containerType) -> LogicalResult {
842  return verifyStructIndices(containerType.getElementType(), indexPos + 1,
843  indices, emitOpError);
844  })
845  .Default([&](auto otherType) -> LogicalResult {
846  return emitOpError()
847  << "type " << otherType << " cannot be indexed (index #"
848  << indexPos << ")";
849  });
850 }
851 
852 /// Driver function around `verifyStructIndices`.
853 static LogicalResult
855  function_ref<InFlightDiagnostic()> emitOpError) {
856  return verifyStructIndices(baseGEPType, /*indexPos=*/1, indices, emitOpError);
857 }
858 
859 LogicalResult LLVM::GEPOp::verify() {
860  if (static_cast<size_t>(
861  llvm::count(getRawConstantIndices(), kDynamicIndex)) !=
862  getDynamicIndices().size())
863  return emitOpError("expected as many dynamic indices as specified in '")
864  << getRawConstantIndicesAttrName().getValue() << "'";
865 
866  return verifyStructIndices(getElemType(), getIndices(),
867  [&] { return emitOpError(); });
868 }
869 
870 //===----------------------------------------------------------------------===//
871 // LoadOp
872 //===----------------------------------------------------------------------===//
873 
874 void LoadOp::getEffects(
876  &effects) {
877  effects.emplace_back(MemoryEffects::Read::get(), &getAddrMutable());
878  // Volatile operations can have target-specific read-write effects on
879  // memory besides the one referred to by the pointer operand.
880  // Similarly, atomic operations that are monotonic or stricter cause
881  // synchronization that from a language point-of-view, are arbitrary
882  // read-writes into memory.
883  if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
884  getOrdering() != AtomicOrdering::unordered)) {
885  effects.emplace_back(MemoryEffects::Write::get());
886  effects.emplace_back(MemoryEffects::Read::get());
887  }
888 }
889 
890 /// Returns true if the given type is supported by atomic operations. All
891 /// integer, float, and pointer types with a power-of-two bitsize and a minimal
892 /// size of 8 bits are supported.
894  const DataLayout &dataLayout) {
895  if (!isa<IntegerType, LLVMPointerType>(type))
897  return false;
898 
899  llvm::TypeSize bitWidth = dataLayout.getTypeSizeInBits(type);
900  if (bitWidth.isScalable())
901  return false;
902  // Needs to be at least 8 bits and a power of two.
903  return bitWidth >= 8 && (bitWidth & (bitWidth - 1)) == 0;
904 }
905 
906 /// Verifies the attributes and the type of atomic memory access operations.
907 template <typename OpTy>
908 LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType,
909  ArrayRef<AtomicOrdering> unsupportedOrderings) {
910  if (memOp.getOrdering() != AtomicOrdering::not_atomic) {
911  DataLayout dataLayout = DataLayout::closest(memOp);
912  if (!isTypeCompatibleWithAtomicOp(valueType, dataLayout))
913  return memOp.emitOpError("unsupported type ")
914  << valueType << " for atomic access";
915  if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering()))
916  return memOp.emitOpError("unsupported ordering '")
917  << stringifyAtomicOrdering(memOp.getOrdering()) << "'";
918  if (!memOp.getAlignment())
919  return memOp.emitOpError("expected alignment for atomic access");
920  return success();
921  }
922  if (memOp.getSyncscope())
923  return memOp.emitOpError(
924  "expected syncscope to be null for non-atomic access");
925  return success();
926 }
927 
928 LogicalResult LoadOp::verify() {
929  Type valueType = getResult().getType();
930  return verifyAtomicMemOp(*this, valueType,
931  {AtomicOrdering::release, AtomicOrdering::acq_rel});
932 }
933 
934 void LoadOp::build(OpBuilder &builder, OperationState &state, Type type,
935  Value addr, unsigned alignment, bool isVolatile,
936  bool isNonTemporal, bool isInvariant, bool isInvariantGroup,
937  AtomicOrdering ordering, StringRef syncscope) {
938  build(builder, state, type, addr,
939  alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
940  isNonTemporal, isInvariant, isInvariantGroup, ordering,
941  syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
942  /*access_groups=*/nullptr,
943  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr,
944  /*tbaa=*/nullptr);
945 }
946 
947 //===----------------------------------------------------------------------===//
948 // StoreOp
949 //===----------------------------------------------------------------------===//
950 
951 void StoreOp::getEffects(
953  &effects) {
954  effects.emplace_back(MemoryEffects::Write::get(), &getAddrMutable());
955  // Volatile operations can have target-specific read-write effects on
956  // memory besides the one referred to by the pointer operand.
957  // Similarly, atomic operations that are monotonic or stricter cause
958  // synchronization that from a language point-of-view, are arbitrary
959  // read-writes into memory.
960  if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
961  getOrdering() != AtomicOrdering::unordered)) {
962  effects.emplace_back(MemoryEffects::Write::get());
963  effects.emplace_back(MemoryEffects::Read::get());
964  }
965 }
966 
967 LogicalResult StoreOp::verify() {
968  Type valueType = getValue().getType();
969  return verifyAtomicMemOp(*this, valueType,
970  {AtomicOrdering::acquire, AtomicOrdering::acq_rel});
971 }
972 
973 void StoreOp::build(OpBuilder &builder, OperationState &state, Value value,
974  Value addr, unsigned alignment, bool isVolatile,
975  bool isNonTemporal, bool isInvariantGroup,
976  AtomicOrdering ordering, StringRef syncscope) {
977  build(builder, state, value, addr,
978  alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
979  isNonTemporal, isInvariantGroup, ordering,
980  syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
981  /*access_groups=*/nullptr,
982  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
983 }
984 
985 //===----------------------------------------------------------------------===//
986 // CallOp
987 //===----------------------------------------------------------------------===//
988 
989 /// Gets the MLIR Op-like result types of a LLVMFunctionType.
990 static SmallVector<Type, 1> getCallOpResultTypes(LLVMFunctionType calleeType) {
991  SmallVector<Type, 1> results;
992  Type resultType = calleeType.getReturnType();
993  if (!isa<LLVM::LLVMVoidType>(resultType))
994  results.push_back(resultType);
995  return results;
996 }
997 
998 /// Gets the variadic callee type for a LLVMFunctionType.
999 static TypeAttr getCallOpVarCalleeType(LLVMFunctionType calleeType) {
1000  return calleeType.isVarArg() ? TypeAttr::get(calleeType) : nullptr;
1001 }
1002 
1003 /// Constructs a LLVMFunctionType from MLIR `results` and `args`.
1004 static LLVMFunctionType getLLVMFuncType(MLIRContext *context, TypeRange results,
1005  ValueRange args) {
1006  Type resultType;
1007  if (results.empty())
1008  resultType = LLVMVoidType::get(context);
1009  else
1010  resultType = results.front();
1011  return LLVMFunctionType::get(resultType, llvm::to_vector(args.getTypes()),
1012  /*isVarArg=*/false);
1013 }
1014 
1015 void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
1016  StringRef callee, ValueRange args) {
1017  build(builder, state, results, builder.getStringAttr(callee), args);
1018 }
1019 
1020 void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
1021  StringAttr callee, ValueRange args) {
1022  build(builder, state, results, SymbolRefAttr::get(callee), args);
1023 }
1024 
1025 void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
1026  FlatSymbolRefAttr callee, ValueRange args) {
1027  assert(callee && "expected non-null callee in direct call builder");
1028  build(builder, state, results,
1029  /*var_callee_type=*/nullptr, callee, args, /*fastmathFlags=*/nullptr,
1030  /*branch_weights=*/nullptr,
1031  /*CConv=*/nullptr, /*TailCallKind=*/nullptr,
1032  /*memory_effects=*/nullptr,
1033  /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
1034  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1035  /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
1036  /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
1037 }
1038 
1039 void CallOp::build(OpBuilder &builder, OperationState &state,
1040  LLVMFunctionType calleeType, StringRef callee,
1041  ValueRange args) {
1042  build(builder, state, calleeType, builder.getStringAttr(callee), args);
1043 }
1044 
1045 void CallOp::build(OpBuilder &builder, OperationState &state,
1046  LLVMFunctionType calleeType, StringAttr callee,
1047  ValueRange args) {
1048  build(builder, state, calleeType, SymbolRefAttr::get(callee), args);
1049 }
1050 
1051 void CallOp::build(OpBuilder &builder, OperationState &state,
1052  LLVMFunctionType calleeType, FlatSymbolRefAttr callee,
1053  ValueRange args) {
1054  build(builder, state, getCallOpResultTypes(calleeType),
1055  getCallOpVarCalleeType(calleeType), callee, args,
1056  /*fastmathFlags=*/nullptr,
1057  /*branch_weights=*/nullptr, /*CConv=*/nullptr,
1058  /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
1059  /*convergent=*/nullptr,
1060  /*no_unwind=*/nullptr, /*will_return=*/nullptr,
1061  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1062  /*access_groups=*/nullptr,
1063  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
1064 }
1065 
1066 void CallOp::build(OpBuilder &builder, OperationState &state,
1067  LLVMFunctionType calleeType, ValueRange args) {
1068  build(builder, state, getCallOpResultTypes(calleeType),
1069  getCallOpVarCalleeType(calleeType),
1070  /*callee=*/nullptr, args,
1071  /*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
1072  /*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
1073  /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
1074  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1075  /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
1076  /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
1077 }
1078 
1079 void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
1080  ValueRange args) {
1081  auto calleeType = func.getFunctionType();
1082  build(builder, state, getCallOpResultTypes(calleeType),
1083  getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), args,
1084  /*fastmathFlags=*/nullptr, /*branch_weights=*/nullptr,
1085  /*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
1086  /*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
1087  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
1088  /*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
1089  /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
1090 }
1091 
1092 CallInterfaceCallable CallOp::getCallableForCallee() {
1093  // Direct call.
1094  if (FlatSymbolRefAttr calleeAttr = getCalleeAttr())
1095  return calleeAttr;
1096  // Indirect call, callee Value is the first operand.
1097  return getOperand(0);
1098 }
1099 
1100 void CallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
1101  // Direct call.
1102  if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) {
1103  auto symRef = callee.get<SymbolRefAttr>();
1104  return setCalleeAttr(cast<FlatSymbolRefAttr>(symRef));
1105  }
1106  // Indirect call, callee Value is the first operand.
1107  return setOperand(0, callee.get<Value>());
1108 }
1109 
1110 Operation::operand_range CallOp::getArgOperands() {
1111  return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
1112 }
1113 
1114 MutableOperandRange CallOp::getArgOperandsMutable() {
1115  return MutableOperandRange(*this, getCallee().has_value() ? 0 : 1,
1116  getCalleeOperands().size());
1117 }
1118 
1119 /// Verify that an inlinable callsite of a debug-info-bearing function in a
1120 /// debug-info-bearing function has a debug location attached to it. This
1121 /// mirrors an LLVM IR verifier.
1122 static LogicalResult verifyCallOpDebugInfo(CallOp callOp, LLVMFuncOp callee) {
1123  if (callee.isExternal())
1124  return success();
1125  auto parentFunc = callOp->getParentOfType<FunctionOpInterface>();
1126  if (!parentFunc)
1127  return success();
1128 
1129  auto hasSubprogram = [](Operation *op) {
1130  return op->getLoc()
1131  ->findInstanceOf<FusedLocWith<LLVM::DISubprogramAttr>>() !=
1132  nullptr;
1133  };
1134  if (!hasSubprogram(parentFunc) || !hasSubprogram(callee))
1135  return success();
1136  bool containsLoc = !isa<UnknownLoc>(callOp->getLoc());
1137  if (!containsLoc)
1138  return callOp.emitError()
1139  << "inlinable function call in a function with a DISubprogram "
1140  "location must have a debug location";
1141  return success();
1142 }
1143 
1144 /// Verify that the parameter and return types of the variadic callee type match
1145 /// the `callOp` argument and result types.
1146 template <typename OpTy>
1147 LogicalResult verifyCallOpVarCalleeType(OpTy callOp) {
1148  std::optional<LLVMFunctionType> varCalleeType = callOp.getVarCalleeType();
1149  if (!varCalleeType)
1150  return success();
1151 
1152  // Verify the variadic callee type is a variadic function type.
1153  if (!varCalleeType->isVarArg())
1154  return callOp.emitOpError(
1155  "expected var_callee_type to be a variadic function type");
1156 
1157  // Verify the variadic callee type has at most as many parameters as the call
1158  // has argument operands.
1159  if (varCalleeType->getNumParams() > callOp.getArgOperands().size())
1160  return callOp.emitOpError("expected var_callee_type to have at most ")
1161  << callOp.getArgOperands().size() << " parameters";
1162 
1163  // Verify the variadic callee type matches the call argument types.
1164  for (auto [paramType, operand] :
1165  llvm::zip(varCalleeType->getParams(), callOp.getArgOperands()))
1166  if (paramType != operand.getType())
1167  return callOp.emitOpError()
1168  << "var_callee_type parameter type mismatch: " << paramType
1169  << " != " << operand.getType();
1170 
1171  // Verify the variadic callee type matches the call result type.
1172  if (!callOp.getNumResults()) {
1173  if (!isa<LLVMVoidType>(varCalleeType->getReturnType()))
1174  return callOp.emitOpError("expected var_callee_type to return void");
1175  } else {
1176  if (callOp.getResult().getType() != varCalleeType->getReturnType())
1177  return callOp.emitOpError("var_callee_type return type mismatch: ")
1178  << varCalleeType->getReturnType()
1179  << " != " << callOp.getResult().getType();
1180  }
1181  return success();
1182 }
1183 
1184 template <typename OpType>
1185 static LogicalResult verifyOperandBundles(OpType &op) {
1186  OperandRangeRange opBundleOperands = op.getOpBundleOperands();
1187  std::optional<ArrayAttr> opBundleTags = op.getOpBundleTags();
1188 
1189  auto isStringAttr = [](Attribute tagAttr) {
1190  return isa<StringAttr>(tagAttr);
1191  };
1192  if (opBundleTags && !llvm::all_of(*opBundleTags, isStringAttr))
1193  return op.emitError("operand bundle tag must be a StringAttr");
1194 
1195  size_t numOpBundles = opBundleOperands.size();
1196  size_t numOpBundleTags = opBundleTags ? opBundleTags->size() : 0;
1197  if (numOpBundles != numOpBundleTags)
1198  return op.emitError("expected ")
1199  << numOpBundles << " operand bundle tags, but actually got "
1200  << numOpBundleTags;
1201 
1202  return success();
1203 }
1204 
1205 LogicalResult CallOp::verify() { return verifyOperandBundles(*this); }
1206 
1207 LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
1208  if (failed(verifyCallOpVarCalleeType(*this)))
1209  return failure();
1210 
1211  // Type for the callee, we'll get it differently depending if it is a direct
1212  // or indirect call.
1213  Type fnType;
1214 
1215  bool isIndirect = false;
1216 
1217  // If this is an indirect call, the callee attribute is missing.
1218  FlatSymbolRefAttr calleeName = getCalleeAttr();
1219  if (!calleeName) {
1220  isIndirect = true;
1221  if (!getNumOperands())
1222  return emitOpError(
1223  "must have either a `callee` attribute or at least an operand");
1224  auto ptrType = llvm::dyn_cast<LLVMPointerType>(getOperand(0).getType());
1225  if (!ptrType)
1226  return emitOpError("indirect call expects a pointer as callee: ")
1227  << getOperand(0).getType();
1228 
1229  return success();
1230  } else {
1231  Operation *callee =
1232  symbolTable.lookupNearestSymbolFrom(*this, calleeName.getAttr());
1233  if (!callee)
1234  return emitOpError()
1235  << "'" << calleeName.getValue()
1236  << "' does not reference a symbol in the current scope";
1237  auto fn = dyn_cast<LLVMFuncOp>(callee);
1238  if (!fn)
1239  return emitOpError() << "'" << calleeName.getValue()
1240  << "' does not reference a valid LLVM function";
1241 
1242  if (failed(verifyCallOpDebugInfo(*this, fn)))
1243  return failure();
1244  fnType = fn.getFunctionType();
1245  }
1246 
1247  LLVMFunctionType funcType = llvm::dyn_cast<LLVMFunctionType>(fnType);
1248  if (!funcType)
1249  return emitOpError("callee does not have a functional type: ") << fnType;
1250 
1251  if (funcType.isVarArg() && !getVarCalleeType())
1252  return emitOpError() << "missing var_callee_type attribute for vararg call";
1253 
1254  // Verify that the operand and result types match the callee.
1255 
1256  if (!funcType.isVarArg() &&
1257  funcType.getNumParams() != (getCalleeOperands().size() - isIndirect))
1258  return emitOpError() << "incorrect number of operands ("
1259  << (getCalleeOperands().size() - isIndirect)
1260  << ") for callee (expecting: "
1261  << funcType.getNumParams() << ")";
1262 
1263  if (funcType.getNumParams() > (getCalleeOperands().size() - isIndirect))
1264  return emitOpError() << "incorrect number of operands ("
1265  << (getCalleeOperands().size() - isIndirect)
1266  << ") for varargs callee (expecting at least: "
1267  << funcType.getNumParams() << ")";
1268 
1269  for (unsigned i = 0, e = funcType.getNumParams(); i != e; ++i)
1270  if (getOperand(i + isIndirect).getType() != funcType.getParamType(i))
1271  return emitOpError() << "operand type mismatch for operand " << i << ": "
1272  << getOperand(i + isIndirect).getType()
1273  << " != " << funcType.getParamType(i);
1274 
1275  if (getNumResults() == 0 &&
1276  !llvm::isa<LLVM::LLVMVoidType>(funcType.getReturnType()))
1277  return emitOpError() << "expected function call to produce a value";
1278 
1279  if (getNumResults() != 0 &&
1280  llvm::isa<LLVM::LLVMVoidType>(funcType.getReturnType()))
1281  return emitOpError()
1282  << "calling function with void result must not produce values";
1283 
1284  if (getNumResults() > 1)
1285  return emitOpError()
1286  << "expected LLVM function call to produce 0 or 1 result";
1287 
1288  if (getNumResults() && getResult().getType() != funcType.getReturnType())
1289  return emitOpError() << "result type mismatch: " << getResult().getType()
1290  << " != " << funcType.getReturnType();
1291 
1292  return success();
1293 }
1294 
1295 void CallOp::print(OpAsmPrinter &p) {
1296  auto callee = getCallee();
1297  bool isDirect = callee.has_value();
1298 
1299  p << ' ';
1300 
1301  // Print calling convention.
1302  if (getCConv() != LLVM::CConv::C)
1303  p << stringifyCConv(getCConv()) << ' ';
1304 
1305  if (getTailCallKind() != LLVM::TailCallKind::None)
1306  p << tailcallkind::stringifyTailCallKind(getTailCallKind()) << ' ';
1307 
1308  // Print the direct callee if present as a function attribute, or an indirect
1309  // callee (first operand) otherwise.
1310  if (isDirect)
1311  p.printSymbolName(callee.value());
1312  else
1313  p << getOperand(0);
1314 
1315  auto args = getCalleeOperands().drop_front(isDirect ? 0 : 1);
1316  p << '(' << args << ')';
1317 
1318  // Print the variadic callee type if the call is variadic.
1319  if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1320  p << " vararg(" << *varCalleeType << ")";
1321 
1322  if (!getOpBundleOperands().empty()) {
1323  p << " ";
1324  printOpBundles(p, *this, getOpBundleOperands(),
1325  getOpBundleOperands().getTypes(), getOpBundleTags());
1326  }
1327 
1328  p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
1329  {getCalleeAttrName(), getTailCallKindAttrName(),
1330  getVarCalleeTypeAttrName(), getCConvAttrName(),
1331  getOperandSegmentSizesAttrName(),
1332  getOpBundleSizesAttrName(),
1333  getOpBundleTagsAttrName()});
1334 
1335  p << " : ";
1336  if (!isDirect)
1337  p << getOperand(0).getType() << ", ";
1338 
1339  // Reconstruct the function MLIR function type from operand and result types.
1340  p.printFunctionalType(args.getTypes(), getResultTypes());
1341 }
1342 
1343 /// Parses the type of a call operation and resolves the operands if the parsing
1344 /// succeeds. Returns failure otherwise.
1346  OpAsmParser &parser, OperationState &result, bool isDirect,
1348  SMLoc trailingTypesLoc = parser.getCurrentLocation();
1349  SmallVector<Type> types;
1350  if (parser.parseColonTypeList(types))
1351  return failure();
1352 
1353  if (isDirect && types.size() != 1)
1354  return parser.emitError(trailingTypesLoc,
1355  "expected direct call to have 1 trailing type");
1356  if (!isDirect && types.size() != 2)
1357  return parser.emitError(trailingTypesLoc,
1358  "expected indirect call to have 2 trailing types");
1359 
1360  auto funcType = llvm::dyn_cast<FunctionType>(types.pop_back_val());
1361  if (!funcType)
1362  return parser.emitError(trailingTypesLoc,
1363  "expected trailing function type");
1364  if (funcType.getNumResults() > 1)
1365  return parser.emitError(trailingTypesLoc,
1366  "expected function with 0 or 1 result");
1367  if (funcType.getNumResults() == 1 &&
1368  llvm::isa<LLVM::LLVMVoidType>(funcType.getResult(0)))
1369  return parser.emitError(trailingTypesLoc,
1370  "expected a non-void result type");
1371 
1372  // The head element of the types list matches the callee type for
1373  // indirect calls, while the types list is emtpy for direct calls.
1374  // Append the function input types to resolve the call operation
1375  // operands.
1376  llvm::append_range(types, funcType.getInputs());
1377  if (parser.resolveOperands(operands, types, parser.getNameLoc(),
1378  result.operands))
1379  return failure();
1380  if (funcType.getNumResults() != 0)
1381  result.addTypes(funcType.getResults());
1382 
1383  return success();
1384 }
1385 
1386 /// Parses an optional function pointer operand before the call argument list
1387 /// for indirect calls, or stops parsing at the function identifier otherwise.
1388 static ParseResult parseOptionalCallFuncPtr(
1389  OpAsmParser &parser,
1391  OpAsmParser::UnresolvedOperand funcPtrOperand;
1392  OptionalParseResult parseResult = parser.parseOptionalOperand(funcPtrOperand);
1393  if (parseResult.has_value()) {
1394  if (failed(*parseResult))
1395  return *parseResult;
1396  operands.push_back(funcPtrOperand);
1397  }
1398  return success();
1399 }
1400 
1401 static ParseResult resolveOpBundleOperands(
1402  OpAsmParser &parser, SMLoc loc, OperationState &state,
1404  ArrayRef<SmallVector<Type>> opBundleOperandTypes,
1405  StringAttr opBundleSizesAttrName) {
1406  unsigned opBundleIndex = 0;
1407  for (const auto &[operands, types] :
1408  llvm::zip_equal(opBundleOperands, opBundleOperandTypes)) {
1409  if (operands.size() != types.size())
1410  return parser.emitError(loc, "expected ")
1411  << operands.size()
1412  << " types for operand bundle operands for operand bundle #"
1413  << opBundleIndex << ", but actually got " << types.size();
1414  if (parser.resolveOperands(operands, types, loc, state.operands))
1415  return failure();
1416  }
1417 
1418  SmallVector<int32_t> opBundleSizes;
1419  opBundleSizes.reserve(opBundleOperands.size());
1420  for (const auto &operands : opBundleOperands)
1421  opBundleSizes.push_back(operands.size());
1422 
1423  state.addAttribute(
1424  opBundleSizesAttrName,
1425  DenseI32ArrayAttr::get(parser.getContext(), opBundleSizes));
1426 
1427  return success();
1428 }
1429 
1430 // <operation> ::= `llvm.call` (cconv)? (tailcallkind)? (function-id | ssa-use)
1431 // `(` ssa-use-list `)`
1432 // ( `vararg(` var-callee-type `)` )?
1433 // ( `[` op-bundles-list `]` )?
1434 // attribute-dict? `:` (type `,`)? function-type
1435 ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
1436  SymbolRefAttr funcAttr;
1437  TypeAttr varCalleeType;
1440  SmallVector<SmallVector<Type>> opBundleOperandTypes;
1441  ArrayAttr opBundleTags;
1442 
1443  // Default to C Calling Convention if no keyword is provided.
1444  result.addAttribute(
1445  getCConvAttrName(result.name),
1446  CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
1447  parser, result, LLVM::CConv::C)));
1448 
1449  result.addAttribute(
1450  getTailCallKindAttrName(result.name),
1452  parseOptionalLLVMKeyword<TailCallKind>(
1453  parser, result, LLVM::TailCallKind::None)));
1454 
1455  // Parse a function pointer for indirect calls.
1456  if (parseOptionalCallFuncPtr(parser, operands))
1457  return failure();
1458  bool isDirect = operands.empty();
1459 
1460  // Parse a function identifier for direct calls.
1461  if (isDirect)
1462  if (parser.parseAttribute(funcAttr, "callee", result.attributes))
1463  return failure();
1464 
1465  // Parse the function arguments.
1466  if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren))
1467  return failure();
1468 
1469  bool isVarArg = parser.parseOptionalKeyword("vararg").succeeded();
1470  if (isVarArg) {
1471  StringAttr varCalleeTypeAttrName =
1472  CallOp::getVarCalleeTypeAttrName(result.name);
1473  if (parser.parseLParen().failed() ||
1474  parser
1475  .parseAttribute(varCalleeType, varCalleeTypeAttrName,
1476  result.attributes)
1477  .failed() ||
1478  parser.parseRParen().failed())
1479  return failure();
1480  }
1481 
1482  SMLoc opBundlesLoc = parser.getCurrentLocation();
1483  if (std::optional<ParseResult> result = parseOpBundles(
1484  parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
1485  result && failed(*result))
1486  return failure();
1487  if (opBundleTags && !opBundleTags.empty())
1488  result.addAttribute(CallOp::getOpBundleTagsAttrName(result.name).getValue(),
1489  opBundleTags);
1490 
1491  if (parser.parseOptionalAttrDict(result.attributes))
1492  return failure();
1493 
1494  // Parse the trailing type list and resolve the operands.
1495  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands))
1496  return failure();
1497  if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
1498  opBundleOperandTypes,
1499  getOpBundleSizesAttrName(result.name)))
1500  return failure();
1501 
1502  int32_t numOpBundleOperands = 0;
1503  for (const auto &operands : opBundleOperands)
1504  numOpBundleOperands += operands.size();
1505 
1506  result.addAttribute(
1507  CallOp::getOperandSegmentSizeAttr(),
1509  {static_cast<int32_t>(operands.size()), numOpBundleOperands}));
1510  return success();
1511 }
1512 
1513 LLVMFunctionType CallOp::getCalleeFunctionType() {
1514  if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1515  return *varCalleeType;
1516  return getLLVMFuncType(getContext(), getResultTypes(), getArgOperands());
1517 }
1518 
1519 ///===---------------------------------------------------------------------===//
1520 /// LLVM::InvokeOp
1521 ///===---------------------------------------------------------------------===//
1522 
1523 void InvokeOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
1524  ValueRange ops, Block *normal, ValueRange normalOps,
1525  Block *unwind, ValueRange unwindOps) {
1526  auto calleeType = func.getFunctionType();
1527  build(builder, state, getCallOpResultTypes(calleeType),
1528  getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), ops,
1529  normalOps, unwindOps, nullptr, nullptr, {}, {}, normal, unwind);
1530 }
1531 
1532 void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
1533  FlatSymbolRefAttr callee, ValueRange ops, Block *normal,
1534  ValueRange normalOps, Block *unwind,
1535  ValueRange unwindOps) {
1536  build(builder, state, tys,
1537  /*var_callee_type=*/nullptr, callee, ops, normalOps, unwindOps, nullptr,
1538  nullptr, {}, {}, normal, unwind);
1539 }
1540 
1541 void InvokeOp::build(OpBuilder &builder, OperationState &state,
1542  LLVMFunctionType calleeType, FlatSymbolRefAttr callee,
1543  ValueRange ops, Block *normal, ValueRange normalOps,
1544  Block *unwind, ValueRange unwindOps) {
1545  build(builder, state, getCallOpResultTypes(calleeType),
1546  getCallOpVarCalleeType(calleeType), callee, ops, normalOps, unwindOps,
1547  nullptr, nullptr, {}, {}, normal, unwind);
1548 }
1549 
1551  assert(index < getNumSuccessors() && "invalid successor index");
1552  return SuccessorOperands(index == 0 ? getNormalDestOperandsMutable()
1553  : getUnwindDestOperandsMutable());
1554 }
1555 
1556 CallInterfaceCallable InvokeOp::getCallableForCallee() {
1557  // Direct call.
1558  if (FlatSymbolRefAttr calleeAttr = getCalleeAttr())
1559  return calleeAttr;
1560  // Indirect call, callee Value is the first operand.
1561  return getOperand(0);
1562 }
1563 
1564 void InvokeOp::setCalleeFromCallable(CallInterfaceCallable callee) {
1565  // Direct call.
1566  if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) {
1567  auto symRef = callee.get<SymbolRefAttr>();
1568  return setCalleeAttr(cast<FlatSymbolRefAttr>(symRef));
1569  }
1570  // Indirect call, callee Value is the first operand.
1571  return setOperand(0, callee.get<Value>());
1572 }
1573 
1574 Operation::operand_range InvokeOp::getArgOperands() {
1575  return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
1576 }
1577 
1578 MutableOperandRange InvokeOp::getArgOperandsMutable() {
1579  return MutableOperandRange(*this, getCallee().has_value() ? 0 : 1,
1580  getCalleeOperands().size());
1581 }
1582 
1583 LogicalResult InvokeOp::verify() {
1584  if (failed(verifyCallOpVarCalleeType(*this)))
1585  return failure();
1586 
1587  Block *unwindDest = getUnwindDest();
1588  if (unwindDest->empty())
1589  return emitError("must have at least one operation in unwind destination");
1590 
1591  // In unwind destination, first operation must be LandingpadOp
1592  if (!isa<LandingpadOp>(unwindDest->front()))
1593  return emitError("first operation in unwind destination should be a "
1594  "llvm.landingpad operation");
1595 
1596  if (failed(verifyOperandBundles(*this)))
1597  return failure();
1598 
1599  return success();
1600 }
1601 
1602 void InvokeOp::print(OpAsmPrinter &p) {
1603  auto callee = getCallee();
1604  bool isDirect = callee.has_value();
1605 
1606  p << ' ';
1607 
1608  // Print calling convention.
1609  if (getCConv() != LLVM::CConv::C)
1610  p << stringifyCConv(getCConv()) << ' ';
1611 
1612  // Either function name or pointer
1613  if (isDirect)
1614  p.printSymbolName(callee.value());
1615  else
1616  p << getOperand(0);
1617 
1618  p << '(' << getCalleeOperands().drop_front(isDirect ? 0 : 1) << ')';
1619  p << " to ";
1620  p.printSuccessorAndUseList(getNormalDest(), getNormalDestOperands());
1621  p << " unwind ";
1622  p.printSuccessorAndUseList(getUnwindDest(), getUnwindDestOperands());
1623 
1624  // Print the variadic callee type if the invoke is variadic.
1625  if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1626  p << " vararg(" << *varCalleeType << ")";
1627 
1628  if (!getOpBundleOperands().empty()) {
1629  p << " ";
1630  printOpBundles(p, *this, getOpBundleOperands(),
1631  getOpBundleOperands().getTypes(), getOpBundleTags());
1632  }
1633 
1634  p.printOptionalAttrDict((*this)->getAttrs(),
1635  {getCalleeAttrName(), getOperandSegmentSizeAttr(),
1636  getCConvAttrName(), getVarCalleeTypeAttrName(),
1637  getOpBundleSizesAttrName(),
1638  getOpBundleTagsAttrName()});
1639 
1640  p << " : ";
1641  if (!isDirect)
1642  p << getOperand(0).getType() << ", ";
1644  llvm::drop_begin(getCalleeOperands().getTypes(), isDirect ? 0 : 1),
1645  getResultTypes());
1646 }
1647 
1648 // <operation> ::= `llvm.invoke` (cconv)? (function-id | ssa-use)
1649 // `(` ssa-use-list `)`
1650 // `to` bb-id (`[` ssa-use-and-type-list `]`)?
1651 // `unwind` bb-id (`[` ssa-use-and-type-list `]`)?
1652 // ( `vararg(` var-callee-type `)` )?
1653 // ( `[` op-bundles-list `]` )?
1654 // attribute-dict? `:` (type `,`)? function-type
1655 ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
1657  SymbolRefAttr funcAttr;
1658  TypeAttr varCalleeType;
1660  SmallVector<SmallVector<Type>> opBundleOperandTypes;
1661  ArrayAttr opBundleTags;
1662  Block *normalDest, *unwindDest;
1663  SmallVector<Value, 4> normalOperands, unwindOperands;
1664  Builder &builder = parser.getBuilder();
1665 
1666  // Default to C Calling Convention if no keyword is provided.
1667  result.addAttribute(
1668  getCConvAttrName(result.name),
1669  CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
1670  parser, result, LLVM::CConv::C)));
1671 
1672  // Parse a function pointer for indirect calls.
1673  if (parseOptionalCallFuncPtr(parser, operands))
1674  return failure();
1675  bool isDirect = operands.empty();
1676 
1677  // Parse a function identifier for direct calls.
1678  if (isDirect && parser.parseAttribute(funcAttr, "callee", result.attributes))
1679  return failure();
1680 
1681  // Parse the function arguments.
1682  if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) ||
1683  parser.parseKeyword("to") ||
1684  parser.parseSuccessorAndUseList(normalDest, normalOperands) ||
1685  parser.parseKeyword("unwind") ||
1686  parser.parseSuccessorAndUseList(unwindDest, unwindOperands))
1687  return failure();
1688 
1689  bool isVarArg = parser.parseOptionalKeyword("vararg").succeeded();
1690  if (isVarArg) {
1691  StringAttr varCalleeTypeAttrName =
1692  InvokeOp::getVarCalleeTypeAttrName(result.name);
1693  if (parser.parseLParen().failed() ||
1694  parser
1695  .parseAttribute(varCalleeType, varCalleeTypeAttrName,
1696  result.attributes)
1697  .failed() ||
1698  parser.parseRParen().failed())
1699  return failure();
1700  }
1701 
1702  SMLoc opBundlesLoc = parser.getCurrentLocation();
1703  if (std::optional<ParseResult> result = parseOpBundles(
1704  parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
1705  result && failed(*result))
1706  return failure();
1707  if (opBundleTags && !opBundleTags.empty())
1708  result.addAttribute(
1709  InvokeOp::getOpBundleTagsAttrName(result.name).getValue(),
1710  opBundleTags);
1711 
1712  if (parser.parseOptionalAttrDict(result.attributes))
1713  return failure();
1714 
1715  // Parse the trailing type list and resolve the function operands.
1716  if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands))
1717  return failure();
1718  if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
1719  opBundleOperandTypes,
1720  getOpBundleSizesAttrName(result.name)))
1721  return failure();
1722 
1723  result.addSuccessors({normalDest, unwindDest});
1724  result.addOperands(normalOperands);
1725  result.addOperands(unwindOperands);
1726 
1727  int32_t numOpBundleOperands = 0;
1728  for (const auto &operands : opBundleOperands)
1729  numOpBundleOperands += operands.size();
1730 
1731  result.addAttribute(
1732  InvokeOp::getOperandSegmentSizeAttr(),
1733  builder.getDenseI32ArrayAttr({static_cast<int32_t>(operands.size()),
1734  static_cast<int32_t>(normalOperands.size()),
1735  static_cast<int32_t>(unwindOperands.size()),
1736  numOpBundleOperands}));
1737  return success();
1738 }
1739 
1740 LLVMFunctionType InvokeOp::getCalleeFunctionType() {
1741  if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
1742  return *varCalleeType;
1743  return getLLVMFuncType(getContext(), getResultTypes(), getArgOperands());
1744 }
1745 
1746 ///===----------------------------------------------------------------------===//
1747 /// Verifying/Printing/Parsing for LLVM::LandingpadOp.
1748 ///===----------------------------------------------------------------------===//
1749 
1750 LogicalResult LandingpadOp::verify() {
1751  Value value;
1752  if (LLVMFuncOp func = (*this)->getParentOfType<LLVMFuncOp>()) {
1753  if (!func.getPersonality())
1754  return emitError(
1755  "llvm.landingpad needs to be in a function with a personality");
1756  }
1757 
1758  // Consistency of llvm.landingpad result types is checked in
1759  // LLVMFuncOp::verify().
1760 
1761  if (!getCleanup() && getOperands().empty())
1762  return emitError("landingpad instruction expects at least one clause or "
1763  "cleanup attribute");
1764 
1765  for (unsigned idx = 0, ie = getNumOperands(); idx < ie; idx++) {
1766  value = getOperand(idx);
1767  bool isFilter = llvm::isa<LLVMArrayType>(value.getType());
1768  if (isFilter) {
1769  // FIXME: Verify filter clauses when arrays are appropriately handled
1770  } else {
1771  // catch - global addresses only.
1772  // Bitcast ops should have global addresses as their args.
1773  if (auto bcOp = value.getDefiningOp<BitcastOp>()) {
1774  if (auto addrOp = bcOp.getArg().getDefiningOp<AddressOfOp>())
1775  continue;
1776  return emitError("constant clauses expected").attachNote(bcOp.getLoc())
1777  << "global addresses expected as operand to "
1778  "bitcast used in clauses for landingpad";
1779  }
1780  // ZeroOp and AddressOfOp allowed
1781  if (value.getDefiningOp<ZeroOp>())
1782  continue;
1783  if (value.getDefiningOp<AddressOfOp>())
1784  continue;
1785  return emitError("clause #")
1786  << idx << " is not a known constant - null, addressof, bitcast";
1787  }
1788  }
1789  return success();
1790 }
1791 
1793  p << (getCleanup() ? " cleanup " : " ");
1794 
1795  // Clauses
1796  for (auto value : getOperands()) {
1797  // Similar to llvm - if clause is an array type then it is filter
1798  // clause else catch clause
1799  bool isArrayTy = llvm::isa<LLVMArrayType>(value.getType());
1800  p << '(' << (isArrayTy ? "filter " : "catch ") << value << " : "
1801  << value.getType() << ") ";
1802  }
1803 
1804  p.printOptionalAttrDict((*this)->getAttrs(), {"cleanup"});
1805 
1806  p << ": " << getType();
1807 }
1808 
1809 // <operation> ::= `llvm.landingpad` `cleanup`?
1810 // ((`catch` | `filter`) operand-type ssa-use)* attribute-dict?
1811 ParseResult LandingpadOp::parse(OpAsmParser &parser, OperationState &result) {
1812  // Check for cleanup
1813  if (succeeded(parser.parseOptionalKeyword("cleanup")))
1814  result.addAttribute("cleanup", parser.getBuilder().getUnitAttr());
1815 
1816  // Parse clauses with types
1817  while (succeeded(parser.parseOptionalLParen()) &&
1818  (succeeded(parser.parseOptionalKeyword("filter")) ||
1819  succeeded(parser.parseOptionalKeyword("catch")))) {
1821  Type ty;
1822  if (parser.parseOperand(operand) || parser.parseColon() ||
1823  parser.parseType(ty) ||
1824  parser.resolveOperand(operand, ty, result.operands) ||
1825  parser.parseRParen())
1826  return failure();
1827  }
1828 
1829  Type type;
1830  if (parser.parseColon() || parser.parseType(type))
1831  return failure();
1832 
1833  result.addTypes(type);
1834  return success();
1835 }
1836 
1837 //===----------------------------------------------------------------------===//
1838 // ExtractValueOp
1839 //===----------------------------------------------------------------------===//
1840 
1841 /// Extract the type at `position` in the LLVM IR aggregate type
1842 /// `containerType`. Each element of `position` is an index into a nested
1843 /// aggregate type. Return the resulting type or emit an error.
1845  function_ref<InFlightDiagnostic(StringRef)> emitError, Type containerType,
1846  ArrayRef<int64_t> position) {
1847  Type llvmType = containerType;
1848  if (!isCompatibleType(containerType)) {
1849  emitError("expected LLVM IR Dialect type, got ") << containerType;
1850  return {};
1851  }
1852 
1853  // Infer the element type from the structure type: iteratively step inside the
1854  // type by taking the element type, indexed by the position attribute for
1855  // structures. Check the position index before accessing, it is supposed to
1856  // be in bounds.
1857  for (int64_t idx : position) {
1858  if (auto arrayType = llvm::dyn_cast<LLVMArrayType>(llvmType)) {
1859  if (idx < 0 || static_cast<unsigned>(idx) >= arrayType.getNumElements()) {
1860  emitError("position out of bounds: ") << idx;
1861  return {};
1862  }
1863  llvmType = arrayType.getElementType();
1864  } else if (auto structType = llvm::dyn_cast<LLVMStructType>(llvmType)) {
1865  if (idx < 0 ||
1866  static_cast<unsigned>(idx) >= structType.getBody().size()) {
1867  emitError("position out of bounds: ") << idx;
1868  return {};
1869  }
1870  llvmType = structType.getBody()[idx];
1871  } else {
1872  emitError("expected LLVM IR structure/array type, got: ") << llvmType;
1873  return {};
1874  }
1875  }
1876  return llvmType;
1877 }
1878 
1879 /// Extract the type at `position` in the wrapped LLVM IR aggregate type
1880 /// `containerType`.
1882  ArrayRef<int64_t> position) {
1883  for (int64_t idx : position) {
1884  if (auto structType = llvm::dyn_cast<LLVMStructType>(llvmType))
1885  llvmType = structType.getBody()[idx];
1886  else
1887  llvmType = llvm::cast<LLVMArrayType>(llvmType).getElementType();
1888  }
1889  return llvmType;
1890 }
1891 
1892 OpFoldResult LLVM::ExtractValueOp::fold(FoldAdaptor adaptor) {
1893  auto insertValueOp = getContainer().getDefiningOp<InsertValueOp>();
1894  OpFoldResult result = {};
1895  while (insertValueOp) {
1896  if (getPosition() == insertValueOp.getPosition())
1897  return insertValueOp.getValue();
1898  unsigned min =
1899  std::min(getPosition().size(), insertValueOp.getPosition().size());
1900  // If one is fully prefix of the other, stop propagating back as it will
1901  // miss dependencies. For instance, %3 should not fold to %f0 in the
1902  // following example:
1903  // ```
1904  // %1 = llvm.insertvalue %f0, %0[0, 0] :
1905  // !llvm.array<4 x !llvm.array<4 x f32>>
1906  // %2 = llvm.insertvalue %arr, %1[0] :
1907  // !llvm.array<4 x !llvm.array<4 x f32>>
1908  // %3 = llvm.extractvalue %2[0, 0] : !llvm.array<4 x !llvm.array<4 x f32>>
1909  // ```
1910  if (getPosition().take_front(min) ==
1911  insertValueOp.getPosition().take_front(min))
1912  return result;
1913 
1914  // If neither a prefix, nor the exact position, we can extract out of the
1915  // value being inserted into. Moreover, we can try again if that operand
1916  // is itself an insertvalue expression.
1917  getContainerMutable().assign(insertValueOp.getContainer());
1918  result = getResult();
1919  insertValueOp = insertValueOp.getContainer().getDefiningOp<InsertValueOp>();
1920  }
1921  return result;
1922 }
1923 
1924 LogicalResult ExtractValueOp::verify() {
1925  auto emitError = [this](StringRef msg) { return emitOpError(msg); };
1927  emitError, getContainer().getType(), getPosition());
1928  if (!valueType)
1929  return failure();
1930 
1931  if (getRes().getType() != valueType)
1932  return emitOpError() << "Type mismatch: extracting from "
1933  << getContainer().getType() << " should produce "
1934  << valueType << " but this op returns "
1935  << getRes().getType();
1936  return success();
1937 }
1938 
1939 void ExtractValueOp::build(OpBuilder &builder, OperationState &state,
1940  Value container, ArrayRef<int64_t> position) {
1941  build(builder, state,
1942  getInsertExtractValueElementType(container.getType(), position),
1943  container, builder.getAttr<DenseI64ArrayAttr>(position));
1944 }
1945 
1946 //===----------------------------------------------------------------------===//
1947 // InsertValueOp
1948 //===----------------------------------------------------------------------===//
1949 
1950 /// Infer the value type from the container type and position.
1951 static ParseResult
1953  Type containerType,
1954  DenseI64ArrayAttr position) {
1956  [&](StringRef msg) {
1957  return parser.emitError(parser.getCurrentLocation(), msg);
1958  },
1959  containerType, position.asArrayRef());
1960  return success(!!valueType);
1961 }
1962 
1963 /// Nothing to print for an inferred type.
1965  Operation *op, Type valueType,
1966  Type containerType,
1967  DenseI64ArrayAttr position) {}
1968 
1969 LogicalResult InsertValueOp::verify() {
1970  auto emitError = [this](StringRef msg) { return emitOpError(msg); };
1972  emitError, getContainer().getType(), getPosition());
1973  if (!valueType)
1974  return failure();
1975 
1976  if (getValue().getType() != valueType)
1977  return emitOpError() << "Type mismatch: cannot insert "
1978  << getValue().getType() << " into "
1979  << getContainer().getType();
1980 
1981  return success();
1982 }
1983 
1984 //===----------------------------------------------------------------------===//
1985 // ReturnOp
1986 //===----------------------------------------------------------------------===//
1987 
1988 LogicalResult ReturnOp::verify() {
1989  auto parent = (*this)->getParentOfType<LLVMFuncOp>();
1990  if (!parent)
1991  return success();
1992 
1993  Type expectedType = parent.getFunctionType().getReturnType();
1994  if (llvm::isa<LLVMVoidType>(expectedType)) {
1995  if (!getArg())
1996  return success();
1997  InFlightDiagnostic diag = emitOpError("expected no operands");
1998  diag.attachNote(parent->getLoc()) << "when returning from function";
1999  return diag;
2000  }
2001  if (!getArg()) {
2002  if (llvm::isa<LLVMVoidType>(expectedType))
2003  return success();
2004  InFlightDiagnostic diag = emitOpError("expected 1 operand");
2005  diag.attachNote(parent->getLoc()) << "when returning from function";
2006  return diag;
2007  }
2008  if (expectedType != getArg().getType()) {
2009  InFlightDiagnostic diag = emitOpError("mismatching result types");
2010  diag.attachNote(parent->getLoc()) << "when returning from function";
2011  return diag;
2012  }
2013  return success();
2014 }
2015 
2016 //===----------------------------------------------------------------------===//
2017 // LLVM::AddressOfOp.
2018 //===----------------------------------------------------------------------===//
2019 
2021  Operation *module = op->getParentOp();
2022  while (module && !satisfiesLLVMModule(module))
2023  module = module->getParentOp();
2024  assert(module && "unexpected operation outside of a module");
2025  return module;
2026 }
2027 
2028 GlobalOp AddressOfOp::getGlobal(SymbolTableCollection &symbolTable) {
2029  return dyn_cast_or_null<GlobalOp>(
2030  symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
2031 }
2032 
2033 LLVMFuncOp AddressOfOp::getFunction(SymbolTableCollection &symbolTable) {
2034  return dyn_cast_or_null<LLVMFuncOp>(
2035  symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
2036 }
2037 
2038 LogicalResult
2039 AddressOfOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2040  Operation *symbol =
2041  symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr());
2042 
2043  auto global = dyn_cast_or_null<GlobalOp>(symbol);
2044  auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
2045 
2046  if (!global && !function)
2047  return emitOpError(
2048  "must reference a global defined by 'llvm.mlir.global' or 'llvm.func'");
2049 
2050  LLVMPointerType type = getType();
2051  if (global && global.getAddrSpace() != type.getAddressSpace())
2052  return emitOpError("pointer address space must match address space of the "
2053  "referenced global");
2054 
2055  return success();
2056 }
2057 
2058 // AddressOfOp constant-folds to the global symbol name.
2059 OpFoldResult LLVM::AddressOfOp::fold(FoldAdaptor) {
2060  return getGlobalNameAttr();
2061 }
2062 
2063 //===----------------------------------------------------------------------===//
2064 // Verifier for LLVM::ComdatOp.
2065 //===----------------------------------------------------------------------===//
2066 
2067 void ComdatOp::build(OpBuilder &builder, OperationState &result,
2068  StringRef symName) {
2069  result.addAttribute(getSymNameAttrName(result.name),
2070  builder.getStringAttr(symName));
2071  Region *body = result.addRegion();
2072  body->emplaceBlock();
2073 }
2074 
2075 LogicalResult ComdatOp::verifyRegions() {
2076  Region &body = getBody();
2077  for (Operation &op : body.getOps())
2078  if (!isa<ComdatSelectorOp>(op))
2079  return op.emitError(
2080  "only comdat selector symbols can appear in a comdat region");
2081 
2082  return success();
2083 }
2084 
2085 //===----------------------------------------------------------------------===//
2086 // Builder, printer and verifier for LLVM::GlobalOp.
2087 //===----------------------------------------------------------------------===//
2088 
2089 void GlobalOp::build(OpBuilder &builder, OperationState &result, Type type,
2090  bool isConstant, Linkage linkage, StringRef name,
2091  Attribute value, uint64_t alignment, unsigned addrSpace,
2092  bool dsoLocal, bool threadLocal, SymbolRefAttr comdat,
2094  ArrayRef<Attribute> dbgExprs) {
2095  result.addAttribute(getSymNameAttrName(result.name),
2096  builder.getStringAttr(name));
2097  result.addAttribute(getGlobalTypeAttrName(result.name), TypeAttr::get(type));
2098  if (isConstant)
2099  result.addAttribute(getConstantAttrName(result.name),
2100  builder.getUnitAttr());
2101  if (value)
2102  result.addAttribute(getValueAttrName(result.name), value);
2103  if (dsoLocal)
2104  result.addAttribute(getDsoLocalAttrName(result.name),
2105  builder.getUnitAttr());
2106  if (threadLocal)
2107  result.addAttribute(getThreadLocal_AttrName(result.name),
2108  builder.getUnitAttr());
2109  if (comdat)
2110  result.addAttribute(getComdatAttrName(result.name), comdat);
2111 
2112  // Only add an alignment attribute if the "alignment" input
2113  // is different from 0. The value must also be a power of two, but
2114  // this is tested in GlobalOp::verify, not here.
2115  if (alignment != 0)
2116  result.addAttribute(getAlignmentAttrName(result.name),
2117  builder.getI64IntegerAttr(alignment));
2118 
2119  result.addAttribute(getLinkageAttrName(result.name),
2120  LinkageAttr::get(builder.getContext(), linkage));
2121  if (addrSpace != 0)
2122  result.addAttribute(getAddrSpaceAttrName(result.name),
2123  builder.getI32IntegerAttr(addrSpace));
2124  result.attributes.append(attrs.begin(), attrs.end());
2125 
2126  if (!dbgExprs.empty())
2127  result.addAttribute(getDbgExprsAttrName(result.name),
2128  ArrayAttr::get(builder.getContext(), dbgExprs));
2129 
2130  result.addRegion();
2131 }
2132 
2133 void GlobalOp::print(OpAsmPrinter &p) {
2134  p << ' ' << stringifyLinkage(getLinkage()) << ' ';
2135  StringRef visibility = stringifyVisibility(getVisibility_());
2136  if (!visibility.empty())
2137  p << visibility << ' ';
2138  if (getThreadLocal_())
2139  p << "thread_local ";
2140  if (auto unnamedAddr = getUnnamedAddr()) {
2141  StringRef str = stringifyUnnamedAddr(*unnamedAddr);
2142  if (!str.empty())
2143  p << str << ' ';
2144  }
2145  if (getConstant())
2146  p << "constant ";
2147  p.printSymbolName(getSymName());
2148  p << '(';
2149  if (auto value = getValueOrNull())
2150  p.printAttribute(value);
2151  p << ')';
2152  if (auto comdat = getComdat())
2153  p << " comdat(" << *comdat << ')';
2154 
2155  // Note that the alignment attribute is printed using the
2156  // default syntax here, even though it is an inherent attribute
2157  // (as defined in https://mlir.llvm.org/docs/LangRef/#attributes)
2158  p.printOptionalAttrDict((*this)->getAttrs(),
2159  {SymbolTable::getSymbolAttrName(),
2160  getGlobalTypeAttrName(), getConstantAttrName(),
2161  getValueAttrName(), getLinkageAttrName(),
2162  getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
2163  getVisibility_AttrName(), getComdatAttrName(),
2164  getUnnamedAddrAttrName()});
2165 
2166  // Print the trailing type unless it's a string global.
2167  if (llvm::dyn_cast_or_null<StringAttr>(getValueOrNull()))
2168  return;
2169  p << " : " << getType();
2170 
2171  Region &initializer = getInitializerRegion();
2172  if (!initializer.empty()) {
2173  p << ' ';
2174  p.printRegion(initializer, /*printEntryBlockArgs=*/false);
2175  }
2176 }
2177 
2178 static LogicalResult verifyComdat(Operation *op,
2179  std::optional<SymbolRefAttr> attr) {
2180  if (!attr)
2181  return success();
2182 
2183  auto *comdatSelector = SymbolTable::lookupNearestSymbolFrom(op, *attr);
2184  if (!isa_and_nonnull<ComdatSelectorOp>(comdatSelector))
2185  return op->emitError() << "expected comdat symbol";
2186 
2187  return success();
2188 }
2189 
2190 // operation ::= `llvm.mlir.global` linkage? visibility?
2191 // (`unnamed_addr` | `local_unnamed_addr`)?
2192 // `thread_local`? `constant`? `@` identifier
2193 // `(` attribute? `)` (`comdat(` symbol-ref-id `)`)?
2194 // attribute-list? (`:` type)? region?
2195 //
2196 // The type can be omitted for string attributes, in which case it will be
2197 // inferred from the value of the string as [strlen(value) x i8].
2198 ParseResult GlobalOp::parse(OpAsmParser &parser, OperationState &result) {
2199  MLIRContext *ctx = parser.getContext();
2200  // Parse optional linkage, default to External.
2201  result.addAttribute(getLinkageAttrName(result.name),
2203  ctx, parseOptionalLLVMKeyword<Linkage>(
2204  parser, result, LLVM::Linkage::External)));
2205 
2206  // Parse optional visibility, default to Default.
2207  result.addAttribute(getVisibility_AttrName(result.name),
2208  parser.getBuilder().getI64IntegerAttr(
2209  parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
2210  parser, result, LLVM::Visibility::Default)));
2211 
2212  // Parse optional UnnamedAddr, default to None.
2213  result.addAttribute(getUnnamedAddrAttrName(result.name),
2214  parser.getBuilder().getI64IntegerAttr(
2215  parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
2216  parser, result, LLVM::UnnamedAddr::None)));
2217 
2218  if (succeeded(parser.parseOptionalKeyword("thread_local")))
2219  result.addAttribute(getThreadLocal_AttrName(result.name),
2220  parser.getBuilder().getUnitAttr());
2221 
2222  if (succeeded(parser.parseOptionalKeyword("constant")))
2223  result.addAttribute(getConstantAttrName(result.name),
2224  parser.getBuilder().getUnitAttr());
2225 
2226  StringAttr name;
2227  if (parser.parseSymbolName(name, getSymNameAttrName(result.name),
2228  result.attributes) ||
2229  parser.parseLParen())
2230  return failure();
2231 
2232  Attribute value;
2233  if (parser.parseOptionalRParen()) {
2234  if (parser.parseAttribute(value, getValueAttrName(result.name),
2235  result.attributes) ||
2236  parser.parseRParen())
2237  return failure();
2238  }
2239 
2240  if (succeeded(parser.parseOptionalKeyword("comdat"))) {
2241  SymbolRefAttr comdat;
2242  if (parser.parseLParen() || parser.parseAttribute(comdat) ||
2243  parser.parseRParen())
2244  return failure();
2245 
2246  result.addAttribute(getComdatAttrName(result.name), comdat);
2247  }
2248 
2249  SmallVector<Type, 1> types;
2250  if (parser.parseOptionalAttrDict(result.attributes) ||
2251  parser.parseOptionalColonTypeList(types))
2252  return failure();
2253 
2254  if (types.size() > 1)
2255  return parser.emitError(parser.getNameLoc(), "expected zero or one type");
2256 
2257  Region &initRegion = *result.addRegion();
2258  if (types.empty()) {
2259  if (auto strAttr = llvm::dyn_cast_or_null<StringAttr>(value)) {
2260  MLIRContext *context = parser.getContext();
2261  auto arrayType = LLVM::LLVMArrayType::get(IntegerType::get(context, 8),
2262  strAttr.getValue().size());
2263  types.push_back(arrayType);
2264  } else {
2265  return parser.emitError(parser.getNameLoc(),
2266  "type can only be omitted for string globals");
2267  }
2268  } else {
2269  OptionalParseResult parseResult =
2270  parser.parseOptionalRegion(initRegion, /*arguments=*/{},
2271  /*argTypes=*/{});
2272  if (parseResult.has_value() && failed(*parseResult))
2273  return failure();
2274  }
2275 
2276  result.addAttribute(getGlobalTypeAttrName(result.name),
2277  TypeAttr::get(types[0]));
2278  return success();
2279 }
2280 
2281 static bool isZeroAttribute(Attribute value) {
2282  if (auto intValue = llvm::dyn_cast<IntegerAttr>(value))
2283  return intValue.getValue().isZero();
2284  if (auto fpValue = llvm::dyn_cast<FloatAttr>(value))
2285  return fpValue.getValue().isZero();
2286  if (auto splatValue = llvm::dyn_cast<SplatElementsAttr>(value))
2287  return isZeroAttribute(splatValue.getSplatValue<Attribute>());
2288  if (auto elementsValue = llvm::dyn_cast<ElementsAttr>(value))
2289  return llvm::all_of(elementsValue.getValues<Attribute>(), isZeroAttribute);
2290  if (auto arrayValue = llvm::dyn_cast<ArrayAttr>(value))
2291  return llvm::all_of(arrayValue.getValue(), isZeroAttribute);
2292  return false;
2293 }
2294 
2295 LogicalResult GlobalOp::verify() {
2296  bool validType = isCompatibleOuterType(getType())
2297  ? !llvm::isa<LLVMVoidType, LLVMTokenType,
2298  LLVMMetadataType, LLVMLabelType>(getType())
2299  : llvm::isa<PointerElementTypeInterface>(getType());
2300  if (!validType)
2301  return emitOpError(
2302  "expects type to be a valid element type for an LLVM global");
2303  if ((*this)->getParentOp() && !satisfiesLLVMModule((*this)->getParentOp()))
2304  return emitOpError("must appear at the module level");
2305 
2306  if (auto strAttr = llvm::dyn_cast_or_null<StringAttr>(getValueOrNull())) {
2307  auto type = llvm::dyn_cast<LLVMArrayType>(getType());
2308  IntegerType elementType =
2309  type ? llvm::dyn_cast<IntegerType>(type.getElementType()) : nullptr;
2310  if (!elementType || elementType.getWidth() != 8 ||
2311  type.getNumElements() != strAttr.getValue().size())
2312  return emitOpError(
2313  "requires an i8 array type of the length equal to that of the string "
2314  "attribute");
2315  }
2316 
2317  if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType())) {
2318  if (!targetExtType.hasProperty(LLVMTargetExtType::CanBeGlobal))
2319  return emitOpError()
2320  << "this target extension type cannot be used in a global";
2321 
2322  if (Attribute value = getValueOrNull())
2323  return emitOpError() << "global with target extension type can only be "
2324  "initialized with zero-initializer";
2325  }
2326 
2327  if (getLinkage() == Linkage::Common) {
2328  if (Attribute value = getValueOrNull()) {
2329  if (!isZeroAttribute(value)) {
2330  return emitOpError()
2331  << "expected zero value for '"
2332  << stringifyLinkage(Linkage::Common) << "' linkage";
2333  }
2334  }
2335  }
2336 
2337  if (getLinkage() == Linkage::Appending) {
2338  if (!llvm::isa<LLVMArrayType>(getType())) {
2339  return emitOpError() << "expected array type for '"
2340  << stringifyLinkage(Linkage::Appending)
2341  << "' linkage";
2342  }
2343  }
2344 
2345  if (failed(verifyComdat(*this, getComdat())))
2346  return failure();
2347 
2348  std::optional<uint64_t> alignAttr = getAlignment();
2349  if (alignAttr.has_value()) {
2350  uint64_t value = alignAttr.value();
2351  if (!llvm::isPowerOf2_64(value))
2352  return emitError() << "alignment attribute is not a power of 2";
2353  }
2354 
2355  return success();
2356 }
2357 
2358 LogicalResult GlobalOp::verifyRegions() {
2359  if (Block *b = getInitializerBlock()) {
2360  ReturnOp ret = cast<ReturnOp>(b->getTerminator());
2361  if (ret.operand_type_begin() == ret.operand_type_end())
2362  return emitOpError("initializer region cannot return void");
2363  if (*ret.operand_type_begin() != getType())
2364  return emitOpError("initializer region type ")
2365  << *ret.operand_type_begin() << " does not match global type "
2366  << getType();
2367 
2368  for (Operation &op : *b) {
2369  auto iface = dyn_cast<MemoryEffectOpInterface>(op);
2370  if (!iface || !iface.hasNoEffect())
2371  return op.emitError()
2372  << "ops with side effects not allowed in global initializers";
2373  }
2374 
2375  if (getValueOrNull())
2376  return emitOpError("cannot have both initializer value and region");
2377  }
2378 
2379  return success();
2380 }
2381 
2382 //===----------------------------------------------------------------------===//
2383 // LLVM::GlobalCtorsOp
2384 //===----------------------------------------------------------------------===//
2385 
2386 LogicalResult
2387 GlobalCtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2388  for (Attribute ctor : getCtors()) {
2389  if (failed(verifySymbolAttrUse(llvm::cast<FlatSymbolRefAttr>(ctor), *this,
2390  symbolTable)))
2391  return failure();
2392  }
2393  return success();
2394 }
2395 
2396 LogicalResult GlobalCtorsOp::verify() {
2397  if (getCtors().size() != getPriorities().size())
2398  return emitError(
2399  "mismatch between the number of ctors and the number of priorities");
2400  return success();
2401 }
2402 
2403 //===----------------------------------------------------------------------===//
2404 // LLVM::GlobalDtorsOp
2405 //===----------------------------------------------------------------------===//
2406 
2407 LogicalResult
2408 GlobalDtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
2409  for (Attribute dtor : getDtors()) {
2410  if (failed(verifySymbolAttrUse(llvm::cast<FlatSymbolRefAttr>(dtor), *this,
2411  symbolTable)))
2412  return failure();
2413  }
2414  return success();
2415 }
2416 
2417 LogicalResult GlobalDtorsOp::verify() {
2418  if (getDtors().size() != getPriorities().size())
2419  return emitError(
2420  "mismatch between the number of dtors and the number of priorities");
2421  return success();
2422 }
2423 
2424 //===----------------------------------------------------------------------===//
2425 // ShuffleVectorOp
2426 //===----------------------------------------------------------------------===//
2427 
2428 void ShuffleVectorOp::build(OpBuilder &builder, OperationState &state, Value v1,
2429  Value v2, DenseI32ArrayAttr mask,
2430  ArrayRef<NamedAttribute> attrs) {
2431  auto containerType = v1.getType();
2432  auto vType = LLVM::getVectorType(LLVM::getVectorElementType(containerType),
2433  mask.size(),
2434  LLVM::isScalableVectorType(containerType));
2435  build(builder, state, vType, v1, v2, mask);
2436  state.addAttributes(attrs);
2437 }
2438 
2439 void ShuffleVectorOp::build(OpBuilder &builder, OperationState &state, Value v1,
2440  Value v2, ArrayRef<int32_t> mask) {
2441  build(builder, state, v1, v2, builder.getDenseI32ArrayAttr(mask));
2442 }
2443 
2444 /// Build the result type of a shuffle vector operation.
2445 static ParseResult parseShuffleType(AsmParser &parser, Type v1Type,
2446  Type &resType, DenseI32ArrayAttr mask) {
2447  if (!LLVM::isCompatibleVectorType(v1Type))
2448  return parser.emitError(parser.getCurrentLocation(),
2449  "expected an LLVM compatible vector type");
2450  resType = LLVM::getVectorType(LLVM::getVectorElementType(v1Type), mask.size(),
2451  LLVM::isScalableVectorType(v1Type));
2452  return success();
2453 }
2454 
2455 /// Nothing to do when the result type is inferred.
2456 static void printShuffleType(AsmPrinter &printer, Operation *op, Type v1Type,
2457  Type resType, DenseI32ArrayAttr mask) {}
2458 
2459 LogicalResult ShuffleVectorOp::verify() {
2460  if (LLVM::isScalableVectorType(getV1().getType()) &&
2461  llvm::any_of(getMask(), [](int32_t v) { return v != 0; }))
2462  return emitOpError("expected a splat operation for scalable vectors");
2463  return success();
2464 }
2465 
2466 //===----------------------------------------------------------------------===//
2467 // Implementations for LLVM::LLVMFuncOp.
2468 //===----------------------------------------------------------------------===//
2469 
2470 // Add the entry block to the function.
2471 Block *LLVMFuncOp::addEntryBlock(OpBuilder &builder) {
2472  assert(empty() && "function already has an entry block");
2473  OpBuilder::InsertionGuard g(builder);
2474  Block *entry = builder.createBlock(&getBody());
2475 
2476  // FIXME: Allow passing in proper locations for the entry arguments.
2477  LLVMFunctionType type = getFunctionType();
2478  for (unsigned i = 0, e = type.getNumParams(); i < e; ++i)
2479  entry->addArgument(type.getParamType(i), getLoc());
2480  return entry;
2481 }
2482 
2483 void LLVMFuncOp::build(OpBuilder &builder, OperationState &result,
2484  StringRef name, Type type, LLVM::Linkage linkage,
2485  bool dsoLocal, CConv cconv, SymbolRefAttr comdat,
2487  ArrayRef<DictionaryAttr> argAttrs,
2488  std::optional<uint64_t> functionEntryCount) {
2489  result.addRegion();
2491  builder.getStringAttr(name));
2492  result.addAttribute(getFunctionTypeAttrName(result.name),
2493  TypeAttr::get(type));
2494  result.addAttribute(getLinkageAttrName(result.name),
2495  LinkageAttr::get(builder.getContext(), linkage));
2496  result.addAttribute(getCConvAttrName(result.name),
2497  CConvAttr::get(builder.getContext(), cconv));
2498  result.attributes.append(attrs.begin(), attrs.end());
2499  if (dsoLocal)
2500  result.addAttribute(getDsoLocalAttrName(result.name),
2501  builder.getUnitAttr());
2502  if (comdat)
2503  result.addAttribute(getComdatAttrName(result.name), comdat);
2504  if (functionEntryCount)
2505  result.addAttribute(getFunctionEntryCountAttrName(result.name),
2506  builder.getI64IntegerAttr(functionEntryCount.value()));
2507  if (argAttrs.empty())
2508  return;
2509 
2510  assert(llvm::cast<LLVMFunctionType>(type).getNumParams() == argAttrs.size() &&
2511  "expected as many argument attribute lists as arguments");
2513  builder, result, argAttrs, /*resultAttrs=*/std::nullopt,
2514  getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
2515 }
2516 
2517 // Builds an LLVM function type from the given lists of input and output types.
2518 // Returns a null type if any of the types provided are non-LLVM types, or if
2519 // there is more than one output type.
2520 static Type
2522  ArrayRef<Type> outputs,
2524  Builder &b = parser.getBuilder();
2525  if (outputs.size() > 1) {
2526  parser.emitError(loc, "failed to construct function type: expected zero or "
2527  "one function result");
2528  return {};
2529  }
2530 
2531  // Convert inputs to LLVM types, exit early on error.
2532  SmallVector<Type, 4> llvmInputs;
2533  for (auto t : inputs) {
2534  if (!isCompatibleType(t)) {
2535  parser.emitError(loc, "failed to construct function type: expected LLVM "
2536  "type for function arguments");
2537  return {};
2538  }
2539  llvmInputs.push_back(t);
2540  }
2541 
2542  // No output is denoted as "void" in LLVM type system.
2543  Type llvmOutput =
2544  outputs.empty() ? LLVMVoidType::get(b.getContext()) : outputs.front();
2545  if (!isCompatibleType(llvmOutput)) {
2546  parser.emitError(loc, "failed to construct function type: expected LLVM "
2547  "type for function results")
2548  << llvmOutput;
2549  return {};
2550  }
2551  return LLVMFunctionType::get(llvmOutput, llvmInputs,
2552  variadicFlag.isVariadic());
2553 }
2554 
2555 // Parses an LLVM function.
2556 //
2557 // operation ::= `llvm.func` linkage? cconv? function-signature
2558 // (`comdat(` symbol-ref-id `)`)?
2559 // function-attributes?
2560 // function-body
2561 //
2562 ParseResult LLVMFuncOp::parse(OpAsmParser &parser, OperationState &result) {
2563  // Default to external linkage if no keyword is provided.
2564  result.addAttribute(
2565  getLinkageAttrName(result.name),
2566  LinkageAttr::get(parser.getContext(),
2567  parseOptionalLLVMKeyword<Linkage>(
2568  parser, result, LLVM::Linkage::External)));
2569 
2570  // Parse optional visibility, default to Default.
2571  result.addAttribute(getVisibility_AttrName(result.name),
2572  parser.getBuilder().getI64IntegerAttr(
2573  parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
2574  parser, result, LLVM::Visibility::Default)));
2575 
2576  // Parse optional UnnamedAddr, default to None.
2577  result.addAttribute(getUnnamedAddrAttrName(result.name),
2578  parser.getBuilder().getI64IntegerAttr(
2579  parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
2580  parser, result, LLVM::UnnamedAddr::None)));
2581 
2582  // Default to C Calling Convention if no keyword is provided.
2583  result.addAttribute(
2584  getCConvAttrName(result.name),
2585  CConvAttr::get(parser.getContext(), parseOptionalLLVMKeyword<CConv>(
2586  parser, result, LLVM::CConv::C)));
2587 
2588  StringAttr nameAttr;
2590  SmallVector<DictionaryAttr> resultAttrs;
2591  SmallVector<Type> resultTypes;
2592  bool isVariadic;
2593 
2594  auto signatureLocation = parser.getCurrentLocation();
2595  if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
2596  result.attributes) ||
2598  parser, /*allowVariadic=*/true, entryArgs, isVariadic, resultTypes,
2599  resultAttrs))
2600  return failure();
2601 
2602  SmallVector<Type> argTypes;
2603  for (auto &arg : entryArgs)
2604  argTypes.push_back(arg.type);
2605  auto type =
2606  buildLLVMFunctionType(parser, signatureLocation, argTypes, resultTypes,
2608  if (!type)
2609  return failure();
2610  result.addAttribute(getFunctionTypeAttrName(result.name),
2611  TypeAttr::get(type));
2612 
2613  if (succeeded(parser.parseOptionalKeyword("vscale_range"))) {
2614  int64_t minRange, maxRange;
2615  if (parser.parseLParen() || parser.parseInteger(minRange) ||
2616  parser.parseComma() || parser.parseInteger(maxRange) ||
2617  parser.parseRParen())
2618  return failure();
2619  auto intTy = IntegerType::get(parser.getContext(), 32);
2620  result.addAttribute(
2621  getVscaleRangeAttrName(result.name),
2623  IntegerAttr::get(intTy, minRange),
2624  IntegerAttr::get(intTy, maxRange)));
2625  }
2626  // Parse the optional comdat selector.
2627  if (succeeded(parser.parseOptionalKeyword("comdat"))) {
2628  SymbolRefAttr comdat;
2629  if (parser.parseLParen() || parser.parseAttribute(comdat) ||
2630  parser.parseRParen())
2631  return failure();
2632 
2633  result.addAttribute(getComdatAttrName(result.name), comdat);
2634  }
2635 
2636  if (failed(parser.parseOptionalAttrDictWithKeyword(result.attributes)))
2637  return failure();
2639  parser.getBuilder(), result, entryArgs, resultAttrs,
2640  getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
2641 
2642  auto *body = result.addRegion();
2643  OptionalParseResult parseResult =
2644  parser.parseOptionalRegion(*body, entryArgs);
2645  return failure(parseResult.has_value() && failed(*parseResult));
2646 }
2647 
2648 // Print the LLVMFuncOp. Collects argument and result types and passes them to
2649 // helper functions. Drops "void" result since it cannot be parsed back. Skips
2650 // the external linkage since it is the default value.
2652  p << ' ';
2653  if (getLinkage() != LLVM::Linkage::External)
2654  p << stringifyLinkage(getLinkage()) << ' ';
2655  StringRef visibility = stringifyVisibility(getVisibility_());
2656  if (!visibility.empty())
2657  p << visibility << ' ';
2658  if (auto unnamedAddr = getUnnamedAddr()) {
2659  StringRef str = stringifyUnnamedAddr(*unnamedAddr);
2660  if (!str.empty())
2661  p << str << ' ';
2662  }
2663  if (getCConv() != LLVM::CConv::C)
2664  p << stringifyCConv(getCConv()) << ' ';
2665 
2666  p.printSymbolName(getName());
2667 
2668  LLVMFunctionType fnType = getFunctionType();
2669  SmallVector<Type, 8> argTypes;
2670  SmallVector<Type, 1> resTypes;
2671  argTypes.reserve(fnType.getNumParams());
2672  for (unsigned i = 0, e = fnType.getNumParams(); i < e; ++i)
2673  argTypes.push_back(fnType.getParamType(i));
2674 
2675  Type returnType = fnType.getReturnType();
2676  if (!llvm::isa<LLVMVoidType>(returnType))
2677  resTypes.push_back(returnType);
2678 
2680  isVarArg(), resTypes);
2681 
2682  // Print vscale range if present
2683  if (std::optional<VScaleRangeAttr> vscale = getVscaleRange())
2684  p << " vscale_range(" << vscale->getMinRange().getInt() << ", "
2685  << vscale->getMaxRange().getInt() << ')';
2686 
2687  // Print the optional comdat selector.
2688  if (auto comdat = getComdat())
2689  p << " comdat(" << *comdat << ')';
2690 
2692  p, *this,
2693  {getFunctionTypeAttrName(), getArgAttrsAttrName(), getResAttrsAttrName(),
2694  getLinkageAttrName(), getCConvAttrName(), getVisibility_AttrName(),
2695  getComdatAttrName(), getUnnamedAddrAttrName(),
2696  getVscaleRangeAttrName()});
2697 
2698  // Print the body if this is not an external function.
2699  Region &body = getBody();
2700  if (!body.empty()) {
2701  p << ' ';
2702  p.printRegion(body, /*printEntryBlockArgs=*/false,
2703  /*printBlockTerminators=*/true);
2704  }
2705 }
2706 
2707 // Verifies LLVM- and implementation-specific properties of the LLVM func Op:
2708 // - functions don't have 'common' linkage
2709 // - external functions have 'external' or 'extern_weak' linkage;
2710 // - vararg is (currently) only supported for external functions;
2711 LogicalResult LLVMFuncOp::verify() {
2712  if (getLinkage() == LLVM::Linkage::Common)
2713  return emitOpError() << "functions cannot have '"
2714  << stringifyLinkage(LLVM::Linkage::Common)
2715  << "' linkage";
2716 
2717  if (failed(verifyComdat(*this, getComdat())))
2718  return failure();
2719 
2720  if (isExternal()) {
2721  if (getLinkage() != LLVM::Linkage::External &&
2722  getLinkage() != LLVM::Linkage::ExternWeak)
2723  return emitOpError() << "external functions must have '"
2724  << stringifyLinkage(LLVM::Linkage::External)
2725  << "' or '"
2726  << stringifyLinkage(LLVM::Linkage::ExternWeak)
2727  << "' linkage";
2728  return success();
2729  }
2730 
2731  // In LLVM IR, these attributes are composed by convention, not by design.
2732  if (isNoInline() && isAlwaysInline())
2733  return emitError("no_inline and always_inline attributes are incompatible");
2734 
2735  if (isOptimizeNone() && !isNoInline())
2736  return emitOpError("with optimize_none must also be no_inline");
2737 
2738  Type landingpadResultTy;
2739  StringRef diagnosticMessage;
2740  bool isLandingpadTypeConsistent =
2741  !walk([&](Operation *op) {
2742  const auto checkType = [&](Type type, StringRef errorMessage) {
2743  if (!landingpadResultTy) {
2744  landingpadResultTy = type;
2745  return WalkResult::advance();
2746  }
2747  if (landingpadResultTy != type) {
2748  diagnosticMessage = errorMessage;
2749  return WalkResult::interrupt();
2750  }
2751  return WalkResult::advance();
2752  };
2754  .Case<LandingpadOp>([&](auto landingpad) {
2755  constexpr StringLiteral errorMessage =
2756  "'llvm.landingpad' should have a consistent result type "
2757  "inside a function";
2758  return checkType(landingpad.getType(), errorMessage);
2759  })
2760  .Case<ResumeOp>([&](auto resume) {
2761  constexpr StringLiteral errorMessage =
2762  "'llvm.resume' should have a consistent input type inside a "
2763  "function";
2764  return checkType(resume.getValue().getType(), errorMessage);
2765  })
2766  .Default([](auto) { return WalkResult::skip(); });
2767  }).wasInterrupted();
2768  if (!isLandingpadTypeConsistent) {
2769  assert(!diagnosticMessage.empty() &&
2770  "Expecting a non-empty diagnostic message");
2771  return emitError(diagnosticMessage);
2772  }
2773 
2774  return success();
2775 }
2776 
2777 /// Verifies LLVM- and implementation-specific properties of the LLVM func Op:
2778 /// - entry block arguments are of LLVM types.
2779 LogicalResult LLVMFuncOp::verifyRegions() {
2780  if (isExternal())
2781  return success();
2782 
2783  unsigned numArguments = getFunctionType().getNumParams();
2784  Block &entryBlock = front();
2785  for (unsigned i = 0; i < numArguments; ++i) {
2786  Type argType = entryBlock.getArgument(i).getType();
2787  if (!isCompatibleType(argType))
2788  return emitOpError("entry block argument #")
2789  << i << " is not of LLVM type";
2790  }
2791 
2792  return success();
2793 }
2794 
2795 Region *LLVMFuncOp::getCallableRegion() {
2796  if (isExternal())
2797  return nullptr;
2798  return &getBody();
2799 }
2800 
2801 //===----------------------------------------------------------------------===//
2802 // UndefOp.
2803 //===----------------------------------------------------------------------===//
2804 
2805 /// Fold an undef operation to a dedicated undef attribute.
2806 OpFoldResult LLVM::UndefOp::fold(FoldAdaptor) {
2807  return LLVM::UndefAttr::get(getContext());
2808 }
2809 
2810 //===----------------------------------------------------------------------===//
2811 // PoisonOp.
2812 //===----------------------------------------------------------------------===//
2813 
2814 /// Fold a poison operation to a dedicated poison attribute.
2815 OpFoldResult LLVM::PoisonOp::fold(FoldAdaptor) {
2817 }
2818 
2819 //===----------------------------------------------------------------------===//
2820 // ZeroOp.
2821 //===----------------------------------------------------------------------===//
2822 
2823 LogicalResult LLVM::ZeroOp::verify() {
2824  if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType()))
2825  if (!targetExtType.hasProperty(LLVM::LLVMTargetExtType::HasZeroInit))
2826  return emitOpError()
2827  << "target extension type does not support zero-initializer";
2828 
2829  return success();
2830 }
2831 
2832 /// Fold a zero operation to a builtin zero attribute when possible and fall
2833 /// back to a dedicated zero attribute.
2834 OpFoldResult LLVM::ZeroOp::fold(FoldAdaptor) {
2836  if (result)
2837  return result;
2838  return LLVM::ZeroAttr::get(getContext());
2839 }
2840 
2841 //===----------------------------------------------------------------------===//
2842 // ConstantOp.
2843 //===----------------------------------------------------------------------===//
2844 
2845 /// Compute the total number of elements in the given type, also taking into
2846 /// account nested types. Supported types are `VectorType`, `LLVMArrayType` and
2847 /// `LLVMFixedVectorType`. Everything else is treated as a scalar.
2848 static int64_t getNumElements(Type t) {
2849  if (auto vecType = dyn_cast<VectorType>(t))
2850  return vecType.getNumElements() * getNumElements(vecType.getElementType());
2851  if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(t))
2852  return arrayType.getNumElements() *
2853  getNumElements(arrayType.getElementType());
2854  if (auto vecType = dyn_cast<LLVMFixedVectorType>(t))
2855  return vecType.getNumElements() * getNumElements(vecType.getElementType());
2856  assert(!isa<LLVM::LLVMScalableVectorType>(t) &&
2857  "number of elements of a scalable vector type is unknown");
2858  return 1;
2859 }
2860 
2861 /// Check if the given type is a scalable vector type or a vector/array type
2862 /// that contains a nested scalable vector type.
2863 static bool hasScalableVectorType(Type t) {
2864  if (isa<LLVM::LLVMScalableVectorType>(t))
2865  return true;
2866  if (auto vecType = dyn_cast<VectorType>(t)) {
2867  if (vecType.isScalable())
2868  return true;
2869  return hasScalableVectorType(vecType.getElementType());
2870  }
2871  if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(t))
2872  return hasScalableVectorType(arrayType.getElementType());
2873  if (auto vecType = dyn_cast<LLVMFixedVectorType>(t))
2874  return hasScalableVectorType(vecType.getElementType());
2875  return false;
2876 }
2877 
2878 LogicalResult LLVM::ConstantOp::verify() {
2879  if (StringAttr sAttr = llvm::dyn_cast<StringAttr>(getValue())) {
2880  auto arrayType = llvm::dyn_cast<LLVMArrayType>(getType());
2881  if (!arrayType || arrayType.getNumElements() != sAttr.getValue().size() ||
2882  !arrayType.getElementType().isInteger(8)) {
2883  return emitOpError() << "expected array type of "
2884  << sAttr.getValue().size()
2885  << " i8 elements for the string constant";
2886  }
2887  return success();
2888  }
2889  if (auto structType = dyn_cast<LLVMStructType>(getType())) {
2890  auto arrayAttr = dyn_cast<ArrayAttr>(getValue());
2891  if (!arrayAttr) {
2892  return emitOpError() << "expected array attribute for a struct constant";
2893  }
2894 
2895  ArrayRef<Type> elementTypes = structType.getBody();
2896  if (arrayAttr.size() != elementTypes.size()) {
2897  return emitOpError() << "expected array attribute of size "
2898  << elementTypes.size();
2899  }
2900  for (auto elementTy : elementTypes) {
2901  if (!isa<IntegerType, FloatType, LLVMPPCFP128Type>(elementTy)) {
2902  return emitOpError() << "expected struct element types to be floating "
2903  "point type or integer type";
2904  }
2905  }
2906 
2907  for (size_t i = 0; i < elementTypes.size(); ++i) {
2908  Attribute element = arrayAttr[i];
2909  if (!isa<IntegerAttr, FloatAttr>(element)) {
2910  return emitOpError()
2911  << "expected struct element attribute types to be floating "
2912  "point type or integer type";
2913  }
2914  auto elementType = cast<TypedAttr>(element).getType();
2915  if (elementType != elementTypes[i]) {
2916  return emitOpError()
2917  << "struct element at index " << i << " is of wrong type";
2918  }
2919  }
2920 
2921  return success();
2922  }
2923  if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType())) {
2924  return emitOpError() << "does not support target extension type.";
2925  }
2926 
2927  // Verification of IntegerAttr, FloatAttr, ElementsAttr, ArrayAttr.
2928  if (auto intAttr = dyn_cast<IntegerAttr>(getValue())) {
2929  if (!llvm::isa<IntegerType>(getType()))
2930  return emitOpError() << "expected integer type";
2931  } else if (auto floatAttr = dyn_cast<FloatAttr>(getValue())) {
2932  const llvm::fltSemantics &sem = floatAttr.getValue().getSemantics();
2933  unsigned floatWidth = APFloat::getSizeInBits(sem);
2934  if (auto floatTy = dyn_cast<FloatType>(getType())) {
2935  if (floatTy.getWidth() != floatWidth) {
2936  return emitOpError() << "expected float type of width " << floatWidth;
2937  }
2938  }
2939  // See the comment for getLLVMConstant for more details about why 8-bit
2940  // floats can be represented by integers.
2941  if (isa<IntegerType>(getType()) && !getType().isInteger(floatWidth)) {
2942  return emitOpError() << "expected integer type of width " << floatWidth;
2943  }
2944  } else if (isa<ElementsAttr, ArrayAttr>(getValue())) {
2945  if (hasScalableVectorType(getType())) {
2946  // The exact number of elements of a scalable vector is unknown, so we
2947  // allow only splat attributes.
2948  auto splatElementsAttr = dyn_cast<SplatElementsAttr>(getValue());
2949  if (!splatElementsAttr)
2950  return emitOpError()
2951  << "scalable vector type requires a splat attribute";
2952  return success();
2953  }
2954  if (!isa<VectorType, LLVM::LLVMArrayType, LLVM::LLVMFixedVectorType>(
2955  getType()))
2956  return emitOpError() << "expected vector or array type";
2957  // The number of elements of the attribute and the type must match.
2958  int64_t attrNumElements;
2959  if (auto elementsAttr = dyn_cast<ElementsAttr>(getValue()))
2960  attrNumElements = elementsAttr.getNumElements();
2961  else
2962  attrNumElements = cast<ArrayAttr>(getValue()).size();
2963  if (getNumElements(getType()) != attrNumElements)
2964  return emitOpError()
2965  << "type and attribute have a different number of elements: "
2966  << getNumElements(getType()) << " vs. " << attrNumElements;
2967  } else {
2968  return emitOpError()
2969  << "only supports integer, float, string or elements attributes";
2970  }
2971 
2972  return success();
2973 }
2974 
2975 bool LLVM::ConstantOp::isBuildableWith(Attribute value, Type type) {
2976  // The value's type must be the same as the provided type.
2977  auto typedAttr = dyn_cast<TypedAttr>(value);
2978  if (!typedAttr || typedAttr.getType() != type || !isCompatibleType(type))
2979  return false;
2980  // The value's type must be an LLVM compatible type.
2981  if (!isCompatibleType(type))
2982  return false;
2983  // TODO: Add support for additional attributes kinds once needed.
2984  return isa<IntegerAttr, FloatAttr, ElementsAttr>(value);
2985 }
2986 
2987 ConstantOp LLVM::ConstantOp::materialize(OpBuilder &builder, Attribute value,
2988  Type type, Location loc) {
2989  if (isBuildableWith(value, type))
2990  return builder.create<LLVM::ConstantOp>(loc, cast<TypedAttr>(value));
2991  return nullptr;
2992 }
2993 
2994 // Constant op constant-folds to its value.
2995 OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); }
2996 
2997 //===----------------------------------------------------------------------===//
2998 // AtomicRMWOp
2999 //===----------------------------------------------------------------------===//
3000 
3001 void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
3002  AtomicBinOp binOp, Value ptr, Value val,
3003  AtomicOrdering ordering, StringRef syncscope,
3004  unsigned alignment, bool isVolatile) {
3005  build(builder, state, val.getType(), binOp, ptr, val, ordering,
3006  !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
3007  alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
3008  /*access_groups=*/nullptr,
3009  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
3010 }
3011 
3012 LogicalResult AtomicRMWOp::verify() {
3013  auto valType = getVal().getType();
3014  if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub ||
3015  getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax) {
3016  if (isCompatibleVectorType(valType)) {
3017  if (isScalableVectorType(valType))
3018  return emitOpError("expected LLVM IR fixed vector type");
3019  Type elemType = getVectorElementType(valType);
3020  if (!isCompatibleFloatingPointType(elemType))
3021  return emitOpError(
3022  "expected LLVM IR floating point type for vector element");
3023  } else if (!isCompatibleFloatingPointType(valType)) {
3024  return emitOpError("expected LLVM IR floating point type");
3025  }
3026  } else if (getBinOp() == AtomicBinOp::xchg) {
3027  DataLayout dataLayout = DataLayout::closest(*this);
3028  if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
3029  return emitOpError("unexpected LLVM IR type for 'xchg' bin_op");
3030  } else {
3031  auto intType = llvm::dyn_cast<IntegerType>(valType);
3032  unsigned intBitWidth = intType ? intType.getWidth() : 0;
3033  if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 &&
3034  intBitWidth != 64)
3035  return emitOpError("expected LLVM IR integer type");
3036  }
3037 
3038  if (static_cast<unsigned>(getOrdering()) <
3039  static_cast<unsigned>(AtomicOrdering::monotonic))
3040  return emitOpError() << "expected at least '"
3041  << stringifyAtomicOrdering(AtomicOrdering::monotonic)
3042  << "' ordering";
3043 
3044  return success();
3045 }
3046 
3047 //===----------------------------------------------------------------------===//
3048 // AtomicCmpXchgOp
3049 //===----------------------------------------------------------------------===//
3050 
3051 /// Returns an LLVM struct type that contains a value type and a boolean type.
3052 static LLVMStructType getValAndBoolStructType(Type valType) {
3053  auto boolType = IntegerType::get(valType.getContext(), 1);
3054  return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType});
3055 }
3056 
3057 void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state,
3058  Value ptr, Value cmp, Value val,
3059  AtomicOrdering successOrdering,
3060  AtomicOrdering failureOrdering, StringRef syncscope,
3061  unsigned alignment, bool isWeak, bool isVolatile) {
3062  build(builder, state, getValAndBoolStructType(val.getType()), ptr, cmp, val,
3063  successOrdering, failureOrdering,
3064  !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
3065  alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak,
3066  isVolatile, /*access_groups=*/nullptr,
3067  /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
3068 }
3069 
3070 LogicalResult AtomicCmpXchgOp::verify() {
3071  auto ptrType = llvm::cast<LLVM::LLVMPointerType>(getPtr().getType());
3072  if (!ptrType)
3073  return emitOpError("expected LLVM IR pointer type for operand #0");
3074  auto valType = getVal().getType();
3075  DataLayout dataLayout = DataLayout::closest(*this);
3076  if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
3077  return emitOpError("unexpected LLVM IR type");
3078  if (getSuccessOrdering() < AtomicOrdering::monotonic ||
3079  getFailureOrdering() < AtomicOrdering::monotonic)
3080  return emitOpError("ordering must be at least 'monotonic'");
3081  if (getFailureOrdering() == AtomicOrdering::release ||
3082  getFailureOrdering() == AtomicOrdering::acq_rel)
3083  return emitOpError("failure ordering cannot be 'release' or 'acq_rel'");
3084  return success();
3085 }
3086 
3087 //===----------------------------------------------------------------------===//
3088 // FenceOp
3089 //===----------------------------------------------------------------------===//
3090 
3091 void FenceOp::build(OpBuilder &builder, OperationState &state,
3092  AtomicOrdering ordering, StringRef syncscope) {
3093  build(builder, state, ordering,
3094  syncscope.empty() ? nullptr : builder.getStringAttr(syncscope));
3095 }
3096 
3097 LogicalResult FenceOp::verify() {
3098  if (getOrdering() == AtomicOrdering::not_atomic ||
3099  getOrdering() == AtomicOrdering::unordered ||
3100  getOrdering() == AtomicOrdering::monotonic)
3101  return emitOpError("can be given only acquire, release, acq_rel, "
3102  "and seq_cst orderings");
3103  return success();
3104 }
3105 
3106 //===----------------------------------------------------------------------===//
3107 // Verifier for extension ops
3108 //===----------------------------------------------------------------------===//
3109 
3110 /// Verifies that the given extension operation operates on consistent scalars
3111 /// or vectors, and that the target width is larger than the input width.
3112 template <class ExtOp>
3113 static LogicalResult verifyExtOp(ExtOp op) {
3114  IntegerType inputType, outputType;
3115  if (isCompatibleVectorType(op.getArg().getType())) {
3116  if (!isCompatibleVectorType(op.getResult().getType()))
3117  return op.emitError(
3118  "input type is a vector but output type is an integer");
3119  if (getVectorNumElements(op.getArg().getType()) !=
3120  getVectorNumElements(op.getResult().getType()))
3121  return op.emitError("input and output vectors are of incompatible shape");
3122  // Because this is a CastOp, the element of vectors is guaranteed to be an
3123  // integer.
3124  inputType = cast<IntegerType>(getVectorElementType(op.getArg().getType()));
3125  outputType =
3126  cast<IntegerType>(getVectorElementType(op.getResult().getType()));
3127  } else {
3128  // Because this is a CastOp and arg is not a vector, arg is guaranteed to be
3129  // an integer.
3130  inputType = cast<IntegerType>(op.getArg().getType());
3131  outputType = dyn_cast<IntegerType>(op.getResult().getType());
3132  if (!outputType)
3133  return op.emitError(
3134  "input type is an integer but output type is a vector");
3135  }
3136 
3137  if (outputType.getWidth() <= inputType.getWidth())
3138  return op.emitError("integer width of the output type is smaller or "
3139  "equal to the integer width of the input type");
3140  return success();
3141 }
3142 
3143 //===----------------------------------------------------------------------===//
3144 // ZExtOp
3145 //===----------------------------------------------------------------------===//
3146 
3147 LogicalResult ZExtOp::verify() { return verifyExtOp<ZExtOp>(*this); }
3148 
3149 OpFoldResult LLVM::ZExtOp::fold(FoldAdaptor adaptor) {
3150  auto arg = dyn_cast_or_null<IntegerAttr>(adaptor.getArg());
3151  if (!arg)
3152  return {};
3153 
3154  size_t targetSize = cast<IntegerType>(getType()).getWidth();
3155  return IntegerAttr::get(getType(), arg.getValue().zext(targetSize));
3156 }
3157 
3158 //===----------------------------------------------------------------------===//
3159 // SExtOp
3160 //===----------------------------------------------------------------------===//
3161 
3162 LogicalResult SExtOp::verify() { return verifyExtOp<SExtOp>(*this); }
3163 
3164 //===----------------------------------------------------------------------===//
3165 // Folder and verifier for LLVM::BitcastOp
3166 //===----------------------------------------------------------------------===//
3167 
3168 /// Folds a cast op that can be chained.
3169 template <typename T>
3171  typename T::FoldAdaptor adaptor) {
3172  // cast(x : T0, T0) -> x
3173  if (castOp.getArg().getType() == castOp.getType())
3174  return castOp.getArg();
3175  if (auto prev = castOp.getArg().template getDefiningOp<T>()) {
3176  // cast(cast(x : T0, T1), T0) -> x
3177  if (prev.getArg().getType() == castOp.getType())
3178  return prev.getArg();
3179  // cast(cast(x : T0, T1), T2) -> cast(x: T0, T2)
3180  castOp.getArgMutable().set(prev.getArg());
3181  return Value{castOp};
3182  }
3183  return {};
3184 }
3185 
3186 OpFoldResult LLVM::BitcastOp::fold(FoldAdaptor adaptor) {
3187  return foldChainableCast(*this, adaptor);
3188 }
3189 
3190 LogicalResult LLVM::BitcastOp::verify() {
3191  auto resultType = llvm::dyn_cast<LLVMPointerType>(
3192  extractVectorElementType(getResult().getType()));
3193  auto sourceType = llvm::dyn_cast<LLVMPointerType>(
3194  extractVectorElementType(getArg().getType()));
3195 
3196  // If one of the types is a pointer (or vector of pointers), then
3197  // both source and result type have to be pointers.
3198  if (static_cast<bool>(resultType) != static_cast<bool>(sourceType))
3199  return emitOpError("can only cast pointers from and to pointers");
3200 
3201  if (!resultType)
3202  return success();
3203 
3204  auto isVector =
3205  llvm::IsaPred<VectorType, LLVMScalableVectorType, LLVMFixedVectorType>;
3206 
3207  // Due to bitcast requiring both operands to be of the same size, it is not
3208  // possible for only one of the two to be a pointer of vectors.
3209  if (isVector(getResult().getType()) && !isVector(getArg().getType()))
3210  return emitOpError("cannot cast pointer to vector of pointers");
3211 
3212  if (!isVector(getResult().getType()) && isVector(getArg().getType()))
3213  return emitOpError("cannot cast vector of pointers to pointer");
3214 
3215  // Bitcast cannot cast between pointers of different address spaces.
3216  // 'llvm.addrspacecast' must be used for this purpose instead.
3217  if (resultType.getAddressSpace() != sourceType.getAddressSpace())
3218  return emitOpError("cannot cast pointers of different address spaces, "
3219  "use 'llvm.addrspacecast' instead");
3220 
3221  return success();
3222 }
3223 
3224 //===----------------------------------------------------------------------===//
3225 // Folder for LLVM::AddrSpaceCastOp
3226 //===----------------------------------------------------------------------===//
3227 
3228 OpFoldResult LLVM::AddrSpaceCastOp::fold(FoldAdaptor adaptor) {
3229  return foldChainableCast(*this, adaptor);
3230 }
3231 
3232 Value LLVM::AddrSpaceCastOp::getViewSource() { return getArg(); }
3233 
3234 //===----------------------------------------------------------------------===//
3235 // Folder for LLVM::GEPOp
3236 //===----------------------------------------------------------------------===//
3237 
3238 OpFoldResult LLVM::GEPOp::fold(FoldAdaptor adaptor) {
3239  GEPIndicesAdaptor<ArrayRef<Attribute>> indices(getRawConstantIndicesAttr(),
3240  adaptor.getDynamicIndices());
3241 
3242  // gep %x:T, 0 -> %x
3243  if (getBase().getType() == getType() && indices.size() == 1)
3244  if (auto integer = llvm::dyn_cast_or_null<IntegerAttr>(indices[0]))
3245  if (integer.getValue().isZero())
3246  return getBase();
3247 
3248  // Canonicalize any dynamic indices of constant value to constant indices.
3249  bool changed = false;
3250  SmallVector<GEPArg> gepArgs;
3251  for (auto iter : llvm::enumerate(indices)) {
3252  auto integer = llvm::dyn_cast_or_null<IntegerAttr>(iter.value());
3253  // Constant indices can only be int32_t, so if integer does not fit we
3254  // are forced to keep it dynamic, despite being a constant.
3255  if (!indices.isDynamicIndex(iter.index()) || !integer ||
3256  !integer.getValue().isSignedIntN(kGEPConstantBitWidth)) {
3257 
3258  PointerUnion<IntegerAttr, Value> existing = getIndices()[iter.index()];
3259  if (Value val = llvm::dyn_cast_if_present<Value>(existing))
3260  gepArgs.emplace_back(val);
3261  else
3262  gepArgs.emplace_back(existing.get<IntegerAttr>().getInt());
3263 
3264  continue;
3265  }
3266 
3267  changed = true;
3268  gepArgs.emplace_back(integer.getInt());
3269  }
3270  if (changed) {
3271  SmallVector<int32_t> rawConstantIndices;
3272  SmallVector<Value> dynamicIndices;
3273  destructureIndices(getElemType(), gepArgs, rawConstantIndices,
3274  dynamicIndices);
3275 
3276  getDynamicIndicesMutable().assign(dynamicIndices);
3277  setRawConstantIndices(rawConstantIndices);
3278  return Value{*this};
3279  }
3280 
3281  return {};
3282 }
3283 
3284 Value LLVM::GEPOp::getViewSource() { return getBase(); }
3285 
3286 //===----------------------------------------------------------------------===//
3287 // ShlOp
3288 //===----------------------------------------------------------------------===//
3289 
3290 OpFoldResult LLVM::ShlOp::fold(FoldAdaptor adaptor) {
3291  auto rhs = dyn_cast_or_null<IntegerAttr>(adaptor.getRhs());
3292  if (!rhs)
3293  return {};
3294 
3295  if (rhs.getValue().getZExtValue() >=
3296  getLhs().getType().getIntOrFloatBitWidth())
3297  return {}; // TODO: Fold into poison.
3298 
3299  auto lhs = dyn_cast_or_null<IntegerAttr>(adaptor.getLhs());
3300  if (!lhs)
3301  return {};
3302 
3303  return IntegerAttr::get(getType(), lhs.getValue().shl(rhs.getValue()));
3304 }
3305 
3306 //===----------------------------------------------------------------------===//
3307 // OrOp
3308 //===----------------------------------------------------------------------===//
3309 
3310 OpFoldResult LLVM::OrOp::fold(FoldAdaptor adaptor) {
3311  auto lhs = dyn_cast_or_null<IntegerAttr>(adaptor.getLhs());
3312  if (!lhs)
3313  return {};
3314 
3315  auto rhs = dyn_cast_or_null<IntegerAttr>(adaptor.getRhs());
3316  if (!rhs)
3317  return {};
3318 
3319  return IntegerAttr::get(getType(), lhs.getValue() | rhs.getValue());
3320 }
3321 
3322 //===----------------------------------------------------------------------===//
3323 // CallIntrinsicOp
3324 //===----------------------------------------------------------------------===//
3325 
3326 LogicalResult CallIntrinsicOp::verify() {
3327  if (!getIntrin().starts_with("llvm."))
3328  return emitOpError() << "intrinsic name must start with 'llvm.'";
3329  if (failed(verifyOperandBundles(*this)))
3330  return failure();
3331  return success();
3332 }
3333 
3334 void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3335  mlir::StringAttr intrin, mlir::ValueRange args) {
3336  build(builder, state, /*resultTypes=*/TypeRange{}, intrin, args,
3337  FastmathFlagsAttr{},
3338  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{});
3339 }
3340 
3341 void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3342  mlir::StringAttr intrin, mlir::ValueRange args,
3343  mlir::LLVM::FastmathFlagsAttr fastMathFlags) {
3344  build(builder, state, /*resultTypes=*/TypeRange{}, intrin, args,
3345  fastMathFlags,
3346  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{});
3347 }
3348 
3349 void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3350  mlir::Type resultType, mlir::StringAttr intrin,
3351  mlir::ValueRange args) {
3352  build(builder, state, {resultType}, intrin, args, FastmathFlagsAttr{},
3353  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{});
3354 }
3355 
3356 void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
3357  mlir::TypeRange resultTypes,
3358  mlir::StringAttr intrin, mlir::ValueRange args,
3359  mlir::LLVM::FastmathFlagsAttr fastMathFlags) {
3360  build(builder, state, resultTypes, intrin, args, fastMathFlags,
3361  /*op_bundle_operands=*/{}, /*op_bundle_tags=*/{});
3362 }
3363 
3364 //===----------------------------------------------------------------------===//
3365 // OpAsmDialectInterface
3366 //===----------------------------------------------------------------------===//
3367 
3368 namespace {
3369 struct LLVMOpAsmDialectInterface : public OpAsmDialectInterface {
3371 
3372  AliasResult getAlias(Attribute attr, raw_ostream &os) const override {
3374  .Case<AccessGroupAttr, AliasScopeAttr, AliasScopeDomainAttr,
3375  DIBasicTypeAttr, DICommonBlockAttr, DICompileUnitAttr,
3376  DICompositeTypeAttr, DIDerivedTypeAttr, DIFileAttr,
3377  DIGlobalVariableAttr, DIGlobalVariableExpressionAttr,
3378  DIImportedEntityAttr, DILabelAttr, DILexicalBlockAttr,
3379  DILexicalBlockFileAttr, DILocalVariableAttr, DIModuleAttr,
3380  DINamespaceAttr, DINullTypeAttr, DIStringTypeAttr,
3381  DISubprogramAttr, DISubroutineTypeAttr, LoopAnnotationAttr,
3382  LoopVectorizeAttr, LoopInterleaveAttr, LoopUnrollAttr,
3383  LoopUnrollAndJamAttr, LoopLICMAttr, LoopDistributeAttr,
3384  LoopPipelineAttr, LoopPeeledAttr, LoopUnswitchAttr, TBAARootAttr,
3385  TBAATagAttr, TBAATypeDescriptorAttr>([&](auto attr) {
3386  os << decltype(attr)::getMnemonic();
3387  return AliasResult::OverridableAlias;
3388  })
3389  .Default([](Attribute) { return AliasResult::NoAlias; });
3390  }
3391 };
3392 } // namespace
3393 
3394 //===----------------------------------------------------------------------===//
3395 // LinkerOptionsOp
3396 //===----------------------------------------------------------------------===//
3397 
3398 LogicalResult LinkerOptionsOp::verify() {
3399  if (mlir::Operation *parentOp = (*this)->getParentOp();
3400  parentOp && !satisfiesLLVMModule(parentOp))
3401  return emitOpError("must appear at the module level");
3402  return success();
3403 }
3404 
3405 //===----------------------------------------------------------------------===//
3406 // InlineAsmOp
3407 //===----------------------------------------------------------------------===//
3408 
3409 void InlineAsmOp::getEffects(
3411  &effects) {
3412  if (getHasSideEffects()) {
3413  effects.emplace_back(MemoryEffects::Write::get());
3414  effects.emplace_back(MemoryEffects::Read::get());
3415  }
3416 }
3417 
3418 //===----------------------------------------------------------------------===//
3419 // AssumeOp (intrinsic)
3420 //===----------------------------------------------------------------------===//
3421 
3422 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3423  mlir::Value cond) {
3424  return build(builder, state, cond, /*op_bundle_operands=*/{},
3425  /*op_bundle_tags=*/ArrayAttr{});
3426 }
3427 
3428 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3429  Value cond,
3430  ArrayRef<llvm::OperandBundleDefT<Value>> opBundles) {
3431  SmallVector<ValueRange> opBundleOperands;
3432  SmallVector<Attribute> opBundleTags;
3433  opBundleOperands.reserve(opBundles.size());
3434  opBundleTags.reserve(opBundles.size());
3435 
3436  for (const llvm::OperandBundleDefT<Value> &bundle : opBundles) {
3437  opBundleOperands.emplace_back(bundle.inputs());
3438  opBundleTags.push_back(
3439  StringAttr::get(builder.getContext(), bundle.getTag()));
3440  }
3441 
3442  auto opBundleTagsAttr = ArrayAttr::get(builder.getContext(), opBundleTags);
3443  return build(builder, state, cond, opBundleOperands, opBundleTagsAttr);
3444 }
3445 
3446 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3447  Value cond, llvm::StringRef tag, ValueRange args) {
3448  llvm::OperandBundleDefT<Value> opBundle(
3449  tag.str(), SmallVector<Value>(args.begin(), args.end()));
3450  return build(builder, state, cond, opBundle);
3451 }
3452 
3453 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3454  Value cond, AssumeAlignTag, Value ptr, Value align) {
3455  return build(builder, state, cond, "align", ValueRange{ptr, align});
3456 }
3457 
3458 void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
3459  Value cond, AssumeSeparateStorageTag, Value ptr1,
3460  Value ptr2) {
3461  return build(builder, state, cond, "separate_storage",
3462  ValueRange{ptr1, ptr2});
3463 }
3464 
3465 LogicalResult LLVM::AssumeOp::verify() { return verifyOperandBundles(*this); }
3466 
3467 //===----------------------------------------------------------------------===//
3468 // masked_gather (intrinsic)
3469 //===----------------------------------------------------------------------===//
3470 
3471 LogicalResult LLVM::masked_gather::verify() {
3472  auto ptrsVectorType = getPtrs().getType();
3473  Type expectedPtrsVectorType =
3475  LLVM::getVectorNumElements(getRes().getType()));
3476  // Vector of pointers type should match result vector type, other than the
3477  // element type.
3478  if (ptrsVectorType != expectedPtrsVectorType)
3479  return emitOpError("expected operand #1 type to be ")
3480  << expectedPtrsVectorType;
3481  return success();
3482 }
3483 
3484 //===----------------------------------------------------------------------===//
3485 // masked_scatter (intrinsic)
3486 //===----------------------------------------------------------------------===//
3487 
3488 LogicalResult LLVM::masked_scatter::verify() {
3489  auto ptrsVectorType = getPtrs().getType();
3490  Type expectedPtrsVectorType =
3492  LLVM::getVectorNumElements(getValue().getType()));
3493  // Vector of pointers type should match value vector type, other than the
3494  // element type.
3495  if (ptrsVectorType != expectedPtrsVectorType)
3496  return emitOpError("expected operand #2 type to be ")
3497  << expectedPtrsVectorType;
3498  return success();
3499 }
3500 
3501 //===----------------------------------------------------------------------===//
3502 // LLVMDialect initialization, type parsing, and registration.
3503 //===----------------------------------------------------------------------===//
3504 
3505 void LLVMDialect::initialize() {
3506  registerAttributes();
3507 
3508  // clang-format off
3509  addTypes<LLVMVoidType,
3510  LLVMPPCFP128Type,
3511  LLVMTokenType,
3512  LLVMLabelType,
3513  LLVMMetadataType>();
3514  // clang-format on
3515  registerTypes();
3516 
3517  addOperations<
3518 #define GET_OP_LIST
3519 #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
3520  ,
3521 #define GET_OP_LIST
3522 #include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
3523  >();
3524 
3525  // Support unknown operations because not all LLVM operations are registered.
3526  allowUnknownOperations();
3527  // clang-format off
3528  addInterfaces<LLVMOpAsmDialectInterface>();
3529  // clang-format on
3530  declarePromisedInterface<DialectInlinerInterface, LLVMDialect>();
3531 }
3532 
3533 #define GET_OP_CLASSES
3534 #include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
3535 
3536 #define GET_OP_CLASSES
3537 #include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
3538 
3539 LogicalResult LLVMDialect::verifyDataLayoutString(
3540  StringRef descr, llvm::function_ref<void(const Twine &)> reportError) {
3541  llvm::Expected<llvm::DataLayout> maybeDataLayout =
3542  llvm::DataLayout::parse(descr);
3543  if (maybeDataLayout)
3544  return success();
3545 
3546  std::string message;
3547  llvm::raw_string_ostream messageStream(message);
3548  llvm::logAllUnhandledErrors(maybeDataLayout.takeError(), messageStream);
3549  reportError("invalid data layout descriptor: " + message);
3550  return failure();
3551 }
3552 
3553 /// Verify LLVM dialect attributes.
3554 LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op,
3555  NamedAttribute attr) {
3556  // If the data layout attribute is present, it must use the LLVM data layout
3557  // syntax. Try parsing it and report errors in case of failure. Users of this
3558  // attribute may assume it is well-formed and can pass it to the (asserting)
3559  // llvm::DataLayout constructor.
3560  if (attr.getName() != LLVM::LLVMDialect::getDataLayoutAttrName())
3561  return success();
3562  if (auto stringAttr = llvm::dyn_cast<StringAttr>(attr.getValue()))
3563  return verifyDataLayoutString(
3564  stringAttr.getValue(),
3565  [op](const Twine &message) { op->emitOpError() << message.str(); });
3566 
3567  return op->emitOpError() << "expected '"
3568  << LLVM::LLVMDialect::getDataLayoutAttrName()
3569  << "' to be a string attributes";
3570 }
3571 
3572 LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op,
3573  Type paramType,
3574  NamedAttribute paramAttr) {
3575  // LLVM attribute may be attached to a result of operation that has not been
3576  // converted to LLVM dialect yet, so the result may have a type with unknown
3577  // representation in LLVM dialect type space. In this case we cannot verify
3578  // whether the attribute may be
3579  bool verifyValueType = isCompatibleType(paramType);
3580  StringAttr name = paramAttr.getName();
3581 
3582  auto checkUnitAttrType = [&]() -> LogicalResult {
3583  if (!llvm::isa<UnitAttr>(paramAttr.getValue()))
3584  return op->emitError() << name << " should be a unit attribute";
3585  return success();
3586  };
3587  auto checkTypeAttrType = [&]() -> LogicalResult {
3588  if (!llvm::isa<TypeAttr>(paramAttr.getValue()))
3589  return op->emitError() << name << " should be a type attribute";
3590  return success();
3591  };
3592  auto checkIntegerAttrType = [&]() -> LogicalResult {
3593  if (!llvm::isa<IntegerAttr>(paramAttr.getValue()))
3594  return op->emitError() << name << " should be an integer attribute";
3595  return success();
3596  };
3597  auto checkPointerType = [&]() -> LogicalResult {
3598  if (!llvm::isa<LLVMPointerType>(paramType))
3599  return op->emitError()
3600  << name << " attribute attached to non-pointer LLVM type";
3601  return success();
3602  };
3603  auto checkIntegerType = [&]() -> LogicalResult {
3604  if (!llvm::isa<IntegerType>(paramType))
3605  return op->emitError()
3606  << name << " attribute attached to non-integer LLVM type";
3607  return success();
3608  };
3609  auto checkPointerTypeMatches = [&]() -> LogicalResult {
3610  if (failed(checkPointerType()))
3611  return failure();
3612 
3613  return success();
3614  };
3615 
3616  // Check a unit attribute that is attached to a pointer value.
3617  if (name == LLVMDialect::getNoAliasAttrName() ||
3618  name == LLVMDialect::getReadonlyAttrName() ||
3619  name == LLVMDialect::getReadnoneAttrName() ||
3620  name == LLVMDialect::getWriteOnlyAttrName() ||
3621  name == LLVMDialect::getNestAttrName() ||
3622  name == LLVMDialect::getNoCaptureAttrName() ||
3623  name == LLVMDialect::getNoFreeAttrName() ||
3624  name == LLVMDialect::getNonNullAttrName()) {
3625  if (failed(checkUnitAttrType()))
3626  return failure();
3627  if (verifyValueType && failed(checkPointerType()))
3628  return failure();
3629  return success();
3630  }
3631 
3632  // Check a type attribute that is attached to a pointer value.
3633  if (name == LLVMDialect::getStructRetAttrName() ||
3634  name == LLVMDialect::getByValAttrName() ||
3635  name == LLVMDialect::getByRefAttrName() ||
3636  name == LLVMDialect::getInAllocaAttrName() ||
3637  name == LLVMDialect::getPreallocatedAttrName()) {
3638  if (failed(checkTypeAttrType()))
3639  return failure();
3640  if (verifyValueType && failed(checkPointerTypeMatches()))
3641  return failure();
3642  return success();
3643  }
3644 
3645  // Check a unit attribute that is attached to an integer value.
3646  if (name == LLVMDialect::getSExtAttrName() ||
3647  name == LLVMDialect::getZExtAttrName()) {
3648  if (failed(checkUnitAttrType()))
3649  return failure();
3650  if (verifyValueType && failed(checkIntegerType()))
3651  return failure();
3652  return success();
3653  }
3654 
3655  // Check an integer attribute that is attached to a pointer value.
3656  if (name == LLVMDialect::getAlignAttrName() ||
3657  name == LLVMDialect::getDereferenceableAttrName() ||
3658  name == LLVMDialect::getDereferenceableOrNullAttrName() ||
3659  name == LLVMDialect::getStackAlignmentAttrName()) {
3660  if (failed(checkIntegerAttrType()))
3661  return failure();
3662  if (verifyValueType && failed(checkPointerType()))
3663  return failure();
3664  return success();
3665  }
3666 
3667  // Check a unit attribute that can be attached to arbitrary types.
3668  if (name == LLVMDialect::getNoUndefAttrName() ||
3669  name == LLVMDialect::getInRegAttrName() ||
3670  name == LLVMDialect::getReturnedAttrName())
3671  return checkUnitAttrType();
3672 
3673  return success();
3674 }
3675 
3676 /// Verify LLVMIR function argument attributes.
3677 LogicalResult LLVMDialect::verifyRegionArgAttribute(Operation *op,
3678  unsigned regionIdx,
3679  unsigned argIdx,
3680  NamedAttribute argAttr) {
3681  auto funcOp = dyn_cast<FunctionOpInterface>(op);
3682  if (!funcOp)
3683  return success();
3684  Type argType = funcOp.getArgumentTypes()[argIdx];
3685 
3686  return verifyParameterAttribute(op, argType, argAttr);
3687 }
3688 
3689 LogicalResult LLVMDialect::verifyRegionResultAttribute(Operation *op,
3690  unsigned regionIdx,
3691  unsigned resIdx,
3692  NamedAttribute resAttr) {
3693  auto funcOp = dyn_cast<FunctionOpInterface>(op);
3694  if (!funcOp)
3695  return success();
3696  Type resType = funcOp.getResultTypes()[resIdx];
3697 
3698  // Check to see if this function has a void return with a result attribute
3699  // to it. It isn't clear what semantics we would assign to that.
3700  if (llvm::isa<LLVMVoidType>(resType))
3701  return op->emitError() << "cannot attach result attributes to functions "
3702  "with a void return";
3703 
3704  // Check to see if this attribute is allowed as a result attribute. Only
3705  // explicitly forbidden LLVM attributes will cause an error.
3706  auto name = resAttr.getName();
3707  if (name == LLVMDialect::getAllocAlignAttrName() ||
3708  name == LLVMDialect::getAllocatedPointerAttrName() ||
3709  name == LLVMDialect::getByValAttrName() ||
3710  name == LLVMDialect::getByRefAttrName() ||
3711  name == LLVMDialect::getInAllocaAttrName() ||
3712  name == LLVMDialect::getNestAttrName() ||
3713  name == LLVMDialect::getNoCaptureAttrName() ||
3714  name == LLVMDialect::getNoFreeAttrName() ||
3715  name == LLVMDialect::getPreallocatedAttrName() ||
3716  name == LLVMDialect::getReadnoneAttrName() ||
3717  name == LLVMDialect::getReadonlyAttrName() ||
3718  name == LLVMDialect::getReturnedAttrName() ||
3719  name == LLVMDialect::getStackAlignmentAttrName() ||
3720  name == LLVMDialect::getStructRetAttrName() ||
3721  name == LLVMDialect::getWriteOnlyAttrName())
3722  return op->emitError() << name << " is not a valid result attribute";
3723  return verifyParameterAttribute(op, resType, resAttr);
3724 }
3725 
3727  Type type, Location loc) {
3728  // If this was folded from an operation other than llvm.mlir.constant, it
3729  // should be materialized as such. Note that an llvm.mlir.zero may fold into
3730  // a builtin zero attribute and thus will materialize as a llvm.mlir.constant.
3731  if (auto symbol = dyn_cast<FlatSymbolRefAttr>(value))
3732  if (isa<LLVM::LLVMPointerType>(type))
3733  return builder.create<LLVM::AddressOfOp>(loc, type, symbol);
3734  if (isa<LLVM::UndefAttr>(value))
3735  return builder.create<LLVM::UndefOp>(loc, type);
3736  if (isa<LLVM::PoisonAttr>(value))
3737  return builder.create<LLVM::PoisonOp>(loc, type);
3738  if (isa<LLVM::ZeroAttr>(value))
3739  return builder.create<LLVM::ZeroOp>(loc, type);
3740  // Otherwise try materializing it as a regular llvm.mlir.constant op.
3741  return LLVM::ConstantOp::materialize(builder, value, type, loc);
3742 }
3743 
3744 //===----------------------------------------------------------------------===//
3745 // Utility functions.
3746 //===----------------------------------------------------------------------===//
3747 
3749  StringRef name, StringRef value,
3750  LLVM::Linkage linkage) {
3751  assert(builder.getInsertionBlock() &&
3752  builder.getInsertionBlock()->getParentOp() &&
3753  "expected builder to point to a block constrained in an op");
3754  auto module =
3755  builder.getInsertionBlock()->getParentOp()->getParentOfType<ModuleOp>();
3756  assert(module && "builder points to an op outside of a module");
3757 
3758  // Create the global at the entry of the module.
3759  OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
3760  MLIRContext *ctx = builder.getContext();
3761  auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
3762  auto global = moduleBuilder.create<LLVM::GlobalOp>(
3763  loc, type, /*isConstant=*/true, linkage, name,
3764  builder.getStringAttr(value), /*alignment=*/0);
3765 
3766  LLVMPointerType ptrType = LLVMPointerType::get(ctx);
3767  // Get the pointer to the first character in the global string.
3768  Value globalPtr =
3769  builder.create<LLVM::AddressOfOp>(loc, ptrType, global.getSymNameAttr());
3770  return builder.create<LLVM::GEPOp>(loc, ptrType, type, globalPtr,
3771  ArrayRef<GEPArg>{0, 0});
3772 }
3773 
3775  return op->hasTrait<OpTrait::SymbolTable>() &&
3777 }
static OperandRange getSuccessorOperands(Block *block, unsigned successorIndex)
Return the operand range used to transfer operands from block to its successor with the given index.
Definition: CFGToSCF.cpp:142
static Value getBase(Value v)
Looks through known "view-like" ops to find the base memref.
static Operation * materializeConstant(Dialect *dialect, OpBuilder &builder, Attribute value, Type type, Location loc)
A utility function used to materialize a constant for a given attribute and type.
Definition: FoldUtils.cpp:50
static MLIRContext * getContext(OpFoldResult val)
static int parseOptionalKeywordAlternative(OpAsmParser &parser, ArrayRef< StringRef > keywords)
LogicalResult verifyCallOpVarCalleeType(OpTy callOp)
Verify that the parameter and return types of the variadic callee type match the callOp argument and ...
static ParseResult parseGEPIndices(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &indices, DenseI32ArrayAttr &rawConstantIndices)
static LogicalResult verifyOperandBundles(OpType &op)
static ParseResult parseCmpOp(OpAsmParser &parser, OperationState &result)
LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType, ArrayRef< AtomicOrdering > unsupportedOrderings)
Verifies the attributes and the type of atomic memory access operations.
static void printOneOpBundle(OpAsmPrinter &p, OperandRange operands, TypeRange operandTypes, StringRef tag)
static LogicalResult verifyComdat(Operation *op, std::optional< SymbolRefAttr > attr)
static RetTy parseOptionalLLVMKeyword(OpAsmParser &parser, OperationState &result, EnumTy defaultValue)
Parse an enum from the keyword, or default to the provided default value.
static LLVMFunctionType getLLVMFuncType(MLIRContext *context, TypeRange results, ValueRange args)
Constructs a LLVMFunctionType from MLIR results and args.
static void printSwitchOpCases(OpAsmPrinter &p, SwitchOp op, Type flagType, DenseIntElementsAttr caseValues, SuccessorRange caseDestinations, OperandRangeRange caseOperands, const TypeRangeRange &caseOperandTypes)
static ParseResult resolveOpBundleOperands(OpAsmParser &parser, SMLoc loc, OperationState &state, ArrayRef< SmallVector< OpAsmParser::UnresolvedOperand >> opBundleOperands, ArrayRef< SmallVector< Type >> opBundleOperandTypes, StringAttr opBundleSizesAttrName)
static ParseResult parseOptionalCallFuncPtr(OpAsmParser &parser, SmallVectorImpl< OpAsmParser::UnresolvedOperand > &operands)
Parses an optional function pointer operand before the call argument list for indirect calls,...
static bool isZeroAttribute(Attribute value)
static void printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp, OperandRange indices, DenseI32ArrayAttr rawConstantIndices)
static LLVMStructType getValAndBoolStructType(Type valType)
Returns an LLVM struct type that contains a value type and a boolean type.
static void printOpBundles(OpAsmPrinter &p, Operation *op, OperandRangeRange opBundleOperands, TypeRangeRange opBundleOperandTypes, std::optional< ArrayAttr > opBundleTags)
static void printShuffleType(AsmPrinter &printer, Operation *op, Type v1Type, Type resType, DenseI32ArrayAttr mask)
Nothing to do when the result type is inferred.
static Type buildLLVMFunctionType(OpAsmParser &parser, SMLoc loc, ArrayRef< Type > inputs, ArrayRef< Type > outputs, function_interface_impl::VariadicFlag variadicFlag)
static auto processFMFAttr(ArrayRef< NamedAttribute > attrs)
static std::optional< ParseResult > parseOpBundles(OpAsmParser &p, SmallVector< SmallVector< OpAsmParser::UnresolvedOperand >> &opBundleOperands, SmallVector< SmallVector< Type >> &opBundleOperandTypes, ArrayAttr &opBundleTags)
static Operation * parentLLVMModule(Operation *op)
static ParseResult parseSwitchOpCases(OpAsmParser &parser, Type flagType, DenseIntElementsAttr &caseValues, SmallVectorImpl< Block * > &caseDestinations, SmallVectorImpl< SmallVector< OpAsmParser::UnresolvedOperand >> &caseOperands, SmallVectorImpl< SmallVector< Type >> &caseOperandTypes)
<cases> ::= [ (case (, case )* )? ] <case> ::= integer : bb-id (( ssa-use-and-type-list ))?
static void printOverflowFlags(AsmPrinter &p, Operation *op, IntegerOverflowFlags flags)
static TypeAttr getCallOpVarCalleeType(LLVMFunctionType calleeType)
Gets the variadic callee type for a LLVMFunctionType.
static Type getInsertExtractValueElementType(function_ref< InFlightDiagnostic(StringRef)> emitError, Type containerType, ArrayRef< int64_t > position)
Extract the type at position in the LLVM IR aggregate type containerType.
static ParseResult parseOneOpBundle(OpAsmParser &p, SmallVector< SmallVector< OpAsmParser::UnresolvedOperand >> &opBundleOperands, SmallVector< SmallVector< Type >> &opBundleOperandTypes, SmallVector< Attribute > &opBundleTags)
static ParseResult parseOverflowFlags(AsmParser &p, IntegerOverflowFlags &flags)
Definition: LLVMDialect.cpp:79
static LogicalResult verifySymbolAttrUse(FlatSymbolRefAttr symbol, Operation *op, SymbolTableCollection &symbolTable)
Verifies symbol's use in op to ensure the symbol is a valid and fully defined llvm....
static Type extractVectorElementType(Type type)
Returns the elemental type of any LLVM-compatible vector type or self.
static bool hasScalableVectorType(Type t)
Check if the given type is a scalable vector type or a vector/array type that contains a nested scala...
static OpFoldResult foldChainableCast(T castOp, typename T::FoldAdaptor adaptor)
Folds a cast op that can be chained.
static void destructureIndices(Type currType, ArrayRef< GEPArg > indices, SmallVectorImpl< int32_t > &rawConstantIndices, SmallVectorImpl< Value > &dynamicIndices)
Destructures the 'indices' parameter into 'rawConstantIndices' and 'dynamicIndices',...
static SmallVector< Type, 1 > getCallOpResultTypes(LLVMFunctionType calleeType)
Gets the MLIR Op-like result types of a LLVMFunctionType.
static Type getI1SameShape(Type type)
Returns a boolean type that has the same shape as type.
static Attribute getBoolAttribute(Type type, MLIRContext *ctx, bool value)
Returns a scalar or vector boolean attribute of the given type.
static LogicalResult verifyCallOpDebugInfo(CallOp callOp, LLVMFuncOp callee)
Verify that an inlinable callsite of a debug-info-bearing function in a debug-info-bearing function h...
static ParseResult parseShuffleType(AsmParser &parser, Type v1Type, Type &resType, DenseI32ArrayAttr mask)
Build the result type of a shuffle vector operation.
static LogicalResult verifyExtOp(ExtOp op)
Verifies that the given extension operation operates on consistent scalars or vectors,...
static constexpr const char kElemTypeAttrName[]
static ParseResult parseInsertExtractValueElementType(AsmParser &parser, Type &valueType, Type containerType, DenseI64ArrayAttr position)
Infer the value type from the container type and position.
static LogicalResult verifyStructIndices(Type baseGEPType, unsigned indexPos, GEPIndicesAdaptor< ValueRange > indices, function_ref< InFlightDiagnostic()> emitOpError)
For the given indices, check if they comply with baseGEPType, especially check against LLVMStructType...
static ParseResult parseCallTypeAndResolveOperands(OpAsmParser &parser, OperationState &result, bool isDirect, ArrayRef< OpAsmParser::UnresolvedOperand > operands)
Parses the type of a call operation and resolves the operands if the parsing succeeds.
static int64_t getNumElements(Type t)
Compute the total number of elements in the given type, also taking into account nested types.
static bool isTypeCompatibleWithAtomicOp(Type type, const DataLayout &dataLayout)
Returns true if the given type is supported by atomic operations.
static void printInsertExtractValueElementType(AsmPrinter &printer, Operation *op, Type valueType, Type containerType, DenseI64ArrayAttr position)
Nothing to print for an inferred type.
#define REGISTER_ENUM_TYPE(Ty)
@ None
static std::string diag(const llvm::Value &value)
static Value min(ImplicitLocOpBuilder &builder, Value value, Value bound)
static void print(spirv::VerCapExtAttr triple, DialectAsmPrinter &printer)
static Type getElementType(Type type, ArrayRef< int32_t > indices, function_ref< InFlightDiagnostic(StringRef)> emitErrorFn)
Walks the given type hierarchy with the given indices, potentially down to component granularity,...
Definition: SPIRVOps.cpp:215
The possible results of an alias query.
Definition: AliasAnalysis.h:26
@ NoAlias
The two locations do not alias at all.
Definition: AliasAnalysis.h:34
This base class exposes generic asm parser hooks, usable across the various derived parsers.
ParseResult parseSymbolName(StringAttr &result)
Parse an -identifier and store it (without the '@' symbol) in a string attribute.
@ Paren
Parens surrounding zero or more operands.
@ None
Zero or more operands with no delimiters.
virtual OptionalParseResult parseOptionalInteger(APInt &result)=0
Parse an optional integer value from the stream.
virtual ParseResult parseColonTypeList(SmallVectorImpl< Type > &result)=0
Parse a colon followed by a type list, which must have at least one type.
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual Builder & getBuilder() const =0
Return a builder which provides useful access to MLIRContext, global objects like types and attribute...
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
Definition: AsmPrinter.cpp:73
virtual ParseResult parseRParen()=0
Parse a ) token.
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
virtual ParseResult parseLSquare()=0
Parse a [ token.
virtual ParseResult parseRSquare()=0
Parse a ] token.
virtual ParseResult parseOptionalColonTypeList(SmallVectorImpl< Type > &result)=0
Parse an optional colon followed by a type list, which if present must have at least one type.
ParseResult parseInteger(IntT &result)
Parse an integer value from the stream.
virtual ParseResult parseOptionalRParen()=0
Parse a ) token if present.
virtual ParseResult parseLess()=0
Parse a '<' token.
ParseResult parseString(std::string *string)
Parse a quoted string token.
virtual ParseResult parseOptionalAttrDictWithKeyword(NamedAttrList &result)=0
Parse a named dictionary into 'result' if the attributes keyword is present.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual ParseResult parseOptionalComma()=0
Parse a , token if present.
virtual ParseResult parseColon()=0
Parse a : token.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseOptionalRSquare()=0
Parse a ] token if present.
virtual ParseResult parseGreater()=0
Parse a '>' token.
virtual ParseResult parseLParen()=0
Parse a ( token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
virtual ParseResult parseComma()=0
Parse a , token.
virtual ParseResult parseOptionalLParen()=0
Parse a ( token if present.
ParseResult parseTypeList(SmallVectorImpl< Type > &result)
Parse a type list.
Definition: AsmPrinter.cpp:77
ParseResult parseKeyword(StringRef keyword)
Parse a given keyword.
virtual ParseResult parseOptionalLSquare()=0
Parse a [ token if present.
virtual ParseResult parseAttribute(Attribute &result, Type type={})=0
Parse an arbitrary attribute of a given type and return it in result.
This base class exposes generic asm printer hooks, usable across the various derived printers.
virtual void printSymbolName(StringRef symbolRef)
Print the given string as a symbol reference, i.e.
virtual void printString(StringRef string)
Print the given string as a quoted string, escaping any special or non-printable characters in it.
virtual void printAttribute(Attribute attr)
Attributes are known-constant values of operations.
Definition: Attributes.h:25
This class provides an abstraction over the different types of ranges over Blocks.
Definition: BlockSupport.h:106
Block represents an ordered list of Operations.
Definition: Block.h:33
bool empty()
Definition: Block.h:148
BlockArgument getArgument(unsigned i)
Definition: Block.h:129
BlockArgument addArgument(Type type, Location loc)
Add one value to the argument list.
Definition: Block.cpp:155
Operation & front()
Definition: Block.h:153
Operation * getParentOp()
Returns the closest surrounding operation that contains this block.
Definition: Block.cpp:33
static BoolAttr get(MLIRContext *context, bool value)
This class is a general helper class for creating context-global objects like types,...
Definition: Builders.h:51
UnitAttr getUnitAttr()
Definition: Builders.cpp:138
IntegerAttr getI32IntegerAttr(int32_t value)
Definition: Builders.cpp:240
DenseI32ArrayAttr getDenseI32ArrayAttr(ArrayRef< int32_t > values)
Definition: Builders.cpp:203
IntegerAttr getI64IntegerAttr(int64_t value)
Definition: Builders.cpp:152
StringAttr getStringAttr(const Twine &bytes)
Definition: Builders.cpp:302
TypedAttr getZeroAttr(Type type)
Definition: Builders.cpp:364
MLIRContext * getContext() const
Definition: Builders.h:56
Attr getAttr(Args &&...args)
Get or construct an instance of the attribute Attr with provided arguments.
Definition: Builders.h:107
The main mechanism for performing data layout queries.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
static DenseElementsAttr get(ShapedType type, ArrayRef< Attribute > values)
Constructs a dense elements attribute from an array of element values.
An attribute that represents a reference to a dense integer vector or tensor object.
static DenseIntElementsAttr get(const ShapedType &type, Arg &&arg)
Get an instance of a DenseIntElementsAttr with the given arguments.
A symbol reference with a reference path containing a single element.
StringRef getValue() const
Returns the name of the held symbol reference.
StringAttr getAttr() const
Returns the name of the held symbol reference as a StringAttr.
This class represents a fused location whose metadata is known to be an instance of the given type.
Definition: Location.h:156
This class represents a diagnostic that is inflight and set to be reported.
Definition: Diagnostics.h:314
Diagnostic & attachNote(std::optional< Location > noteLoc=std::nullopt)
Attaches a note to this diagnostic.
Definition: Diagnostics.h:352
Class used for building a 'llvm.getelementptr'.
Definition: LLVMDialect.h:76
Class used for convenient access and iteration over GEP indices.
Definition: LLVMDialect.h:123
bool isDynamicIndex(size_t index) const
Returns whether the GEP index at the given position is a dynamic index.
Definition: LLVMDialect.h:152
size_t size() const
Returns the amount of indices of the GEPOp.
Definition: LLVMDialect.h:157
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:66
MLIRContext is the top-level object for a collection of MLIR operations.
Definition: MLIRContext.h:60
This class provides a mutable adaptor for a range of operands.
Definition: ValueRange.h:115
std::optional< NamedAttribute > getNamed(StringRef name) const
Return the specified named attribute if present, std::nullopt otherwise.
Attribute erase(StringAttr name)
Erase the attribute with the given name from the list.
Attribute set(StringAttr name, Attribute value)
If the an attribute exists with the specified name, change it to the new value.
void append(StringRef name, Attribute attr)
Add an attribute with the specified name.
NamedAttribute represents a combination of a name and an Attribute value.
Definition: Attributes.h:207
StringAttr getName() const
Return the name of the attribute.
Definition: Attributes.cpp:49
Attribute getValue() const
Return the value of the attribute.
Definition: Attributes.h:221
OpAsmDialectInterface(Dialect *dialect)
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult parseSuccessor(Block *&dest)=0
Parse a single operation successor.
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
virtual OptionalParseResult parseOptionalOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single operand if present.
virtual ParseResult parseSuccessorAndUseList(Block *&dest, SmallVectorImpl< Value > &operands)=0
Parse a single operation successor and its operand list.
virtual OptionalParseResult parseOptionalRegion(Region &region, ArrayRef< Argument > arguments={}, bool enableNameShadowing=false)=0
Parses a region if present.
ParseResult resolveOperands(Operands &&operands, Type type, SmallVectorImpl< Value > &result)
Resolve a list of operands to SSA values, emitting an error on failure, or appending the results to t...
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
virtual ParseResult parseOperandList(SmallVectorImpl< UnresolvedOperand > &result, Delimiter delimiter=Delimiter::None, bool allowResultNumber=true, int requiredOperandCount=-1)=0
Parse zero or more SSA comma-separated operand references with a specified surrounding delimiter,...
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printNewline()=0
Print a newline and indent the printer to the start of the current operation.
virtual void printSuccessorAndUseList(Block *successor, ValueRange succOperands)=0
Print the successor and its operands.
void printOperands(const ContainerType &container)
Print a comma separated list of operands.
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
void printFunctionalType(Operation *op)
Print the complete type of an operation in functional form.
Definition: AsmPrinter.cpp:94
virtual void printRegion(Region &blocks, bool printEntryBlockArgs=true, bool printBlockTerminators=true, bool printEmptyBlock=false)=0
Prints a region.
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
RAII guard to reset the insertion point of the builder when destroyed.
Definition: Builders.h:357
This class helps build Operations.
Definition: Builders.h:216
Listener * getListener() const
Returns the current listener of this builder, or nullptr if this builder doesn't have a listener.
Definition: Builders.h:329
Block * createBlock(Region *parent, Region::iterator insertPt={}, TypeRange argTypes=std::nullopt, ArrayRef< Location > locs=std::nullopt)
Add new block with 'argTypes' arguments and set the insertion point to the end of it.
Definition: Builders.cpp:470
Operation * create(const OperationState &state)
Creates an operation given the fields represented as an OperationState.
Definition: Builders.cpp:497
Block * getInsertionBlock() const
Return the block the current insertion point belongs to.
Definition: Builders.h:451
This class represents a single result from folding an operation.
Definition: OpDefinition.h:268
This class provides the API for ops that are known to be isolated from above.
A trait used to provide symbol table functionalities to a region operation.
Definition: SymbolTable.h:435
This class represents a contiguous range of operand ranges, e.g.
Definition: ValueRange.h:82
This class implements the operand iterators for the Operation class.
Definition: ValueRange.h:42
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
bool hasTrait()
Returns true if the operation was registered with a particular trait, e.g.
Definition: Operation.h:745
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition: Operation.h:234
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:268
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
Definition: Operation.h:238
InFlightDiagnostic emitOpError(const Twine &message={})
Emit an error with the op name prefixed, like "'dim' op " which is convenient for verifiers.
Definition: Operation.cpp:671
This class implements Optional functionality for ParseResult.
Definition: OpDefinition.h:39
ParseResult value() const
Access the internal ParseResult value.
Definition: OpDefinition.h:52
bool has_value() const
Returns true if we contain a valid ParseResult value.
Definition: OpDefinition.h:49
This class contains a list of basic blocks and a link to the parent operation it is attached to.
Definition: Region.h:26
iterator_range< OpIterator > getOps()
Definition: Region.h:172
bool empty()
Definition: Region.h:60
Block & emplaceBlock()
Definition: Region.h:46
This class represents a specific instance of an effect.
static DerivedEffect * get()
Returns a unique instance for the derived effect class.
This class models how operands are forwarded to block arguments in control flow.
This class implements the successor iterators for Block.
Definition: BlockSupport.h:73
This class represents a collection of SymbolTables.
Definition: SymbolTable.h:283
Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
Operation * lookupSymbolIn(Operation *symbolTableOp, StringAttr symbol)
Look up a symbol with the specified name within the specified symbol table operation,...
static StringRef getSymbolAttrName()
Return the name of the attribute used for symbol names.
Definition: SymbolTable.h:76
static Operation * lookupNearestSymbolFrom(Operation *from, StringAttr symbol)
Returns the operation registered with the given symbol name within the closest parent operation of,...
This class provides an abstraction for a range of TypeRange.
Definition: TypeRange.h:92
This class provides an abstraction over the various different ranges of value types.
Definition: TypeRange.h:36
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
Definition: Types.cpp:35
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition: Types.cpp:133
This class provides an abstraction over the different types of ranges over Values.
Definition: ValueRange.h:381
type_range getTypes() const
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition: Value.h:96
Type getType() const
Return the type of this value.
Definition: Value.h:129
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition: Value.cpp:20
static WalkResult skip()
Definition: Visitors.h:52
static WalkResult advance()
Definition: Visitors.h:51
static WalkResult interrupt()
Definition: Visitors.h:50
static DenseArrayAttrImpl get(MLIRContext *context, ArrayRef< int32_t > content)
Builder from ArrayRef<T>.
A named class for passing around the variadic flag.
Value createGlobalString(Location loc, OpBuilder &builder, StringRef name, StringRef value, Linkage linkage)
Create an LLVM global containing the string "value" at the module containing surrounding the insertio...
Type getVectorType(Type elementType, unsigned numElements, bool isScalable=false)
Creates an LLVM dialect-compatible vector type with the given element type and length.
Definition: LLVMTypes.cpp:928
bool isScalableVectorType(Type vectorType)
Returns whether a vector type is scalable or not.
Definition: LLVMTypes.cpp:919
bool isCompatibleVectorType(Type type)
Returns true if the given type is a vector type compatible with the LLVM dialect.
Definition: LLVMTypes.cpp:876
bool isCompatibleOuterType(Type type)
Returns true if the given outer type is compatible with the LLVM dialect without checking its potenti...
Definition: LLVMTypes.cpp:763
bool satisfiesLLVMModule(Operation *op)
LLVM requires some operations to be inside of a Module operation.
llvm::PointerEmbeddedInt< int32_t, kGEPConstantBitWidth > GEPConstantIndex
Wrapper around a int32_t for use in a PointerUnion.
Definition: LLVMDialect.h:70
constexpr int kGEPConstantBitWidth
Bit-width of a 'GEPConstantIndex' within GEPArg.
Definition: LLVMDialect.h:67
bool isCompatibleType(Type type)
Returns true if the given type is compatible with the LLVM dialect.
Definition: LLVMTypes.cpp:858
bool isCompatibleFloatingPointType(Type type)
Returns true if the given type is a floating-point type compatible with the LLVM dialect.
Definition: LLVMTypes.cpp:871
llvm::ElementCount getVectorNumElements(Type type)
Returns the element count of any LLVM-compatible vector type.
Definition: LLVMTypes.cpp:901
Type getVectorElementType(Type type)
Returns the element type of any vector type compatible with the LLVM dialect.
Definition: LLVMTypes.cpp:892
void walk(Operation *op, function_ref< void(Region *)> callback, WalkOrder order)
Walk all of the regions, blocks, or operations nested under (and including) the given operation.
Definition: Visitors.h:136
constexpr void enumerate(std::tuple< Tys... > &tuple, CallbackT &&callback)
Definition: Matchers.h:344
void addArgAndResultAttrs(Builder &builder, OperationState &result, ArrayRef< DictionaryAttr > argAttrs, ArrayRef< DictionaryAttr > resultAttrs, StringAttr argAttrsName, StringAttr resAttrsName)
Adds argument and result attributes, provided as argAttrs and resultAttrs arguments,...
ParseResult parseFunctionSignature(OpAsmParser &parser, bool allowVariadic, SmallVectorImpl< OpAsmParser::Argument > &arguments, bool &isVariadic, SmallVectorImpl< Type > &resultTypes, SmallVectorImpl< DictionaryAttr > &resultAttrs)
Parses a function signature using parser.
void printFunctionAttributes(OpAsmPrinter &p, Operation *op, ArrayRef< StringRef > elided={})
Prints the list of function prefixed with the "attributes" keyword.
void printFunctionSignature(OpAsmPrinter &p, FunctionOpInterface op, ArrayRef< Type > argTypes, bool isVariadic, ArrayRef< Type > resultTypes)
Prints the signature of the function-like operation op.
Operation::operand_range getIndices(Operation *op)
Get the indices that the given load/store operation is operating on.
Definition: Utils.cpp:18
QueryRef parse(llvm::StringRef line, const QuerySession &qs)
Definition: Query.cpp:20
Value constantIndex(OpBuilder &builder, Location loc, int64_t i)
Generates a constant of index type.
Definition: CodegenUtils.h:331
Visibility
This enum describes C++ inheritance visibility.
Definition: Class.h:408
std::string stringify(T &&t)
Generically convert a value to a std::string.
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition: Matchers.h:496
detail::constant_int_value_binder m_ConstantInt(IntegerAttr::ValueType *bind_value)
Matches a constant holding a scalar/vector/tensor integer (splat) and writes the integer value to bin...
Definition: Matchers.h:533
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
Definition: Utils.cpp:305
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...
Attribute convertToAttribute(MLIRContext *ctx, int64_t storage)
Convert the provided int64_t to an IntegerAttr attribute.
Definition: ODSSupport.cpp:32
LogicalResult convertFromAttribute(int64_t &storage, Attribute attr, function_ref< InFlightDiagnostic()> emitError)
Convert an IntegerAttr attribute to an int64_t, or return an error if the attribute isn't an IntegerA...
Definition: ODSSupport.cpp:22
LogicalResult verify(Operation *op, bool verifyRecursively=true)
Perform (potentially expensive) checks of invariants, used to detect compiler bugs,...
Definition: Verifier.cpp:426
A callable is either a symbol, or an SSA value, that is referenced by a call-like operation.
This is the representation of an operand reference.
This represents an operation in an abstracted form, suitable for use with the builder APIs.
SmallVector< Value, 4 > operands
void addOperands(ValueRange newOperands)
void addAttributes(ArrayRef< NamedAttribute > newAttributes)
Add an array of named attributes.
void addAttribute(StringRef name, Attribute attr)
Add an attribute with the specified name.
void addSuccessors(Block *successor)
Adds a successor to the operation sate. successor must not be null.
void addTypes(ArrayRef< Type > newTypes)
NamedAttrList attributes
Region * addRegion()
Create a region that should be attached to the operation.