MLIR  21.0.0git
DataLayoutInterfaces.cpp
Go to the documentation of this file.
1 //===- DataLayoutInterfaces.cpp - Data Layout Interface Implementation ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "mlir/IR/BuiltinDialect.h"
11 #include "mlir/IR/BuiltinOps.h"
12 #include "mlir/IR/BuiltinTypes.h"
13 #include "mlir/IR/Operation.h"
14 
15 #include "llvm/ADT/TypeSwitch.h"
16 #include "llvm/Support/MathExtras.h"
17 
18 using namespace mlir;
19 
20 //===----------------------------------------------------------------------===//
21 // Default implementations
22 //===----------------------------------------------------------------------===//
23 
24 /// Reports that the given type is missing the data layout information and
25 /// exits.
26 [[noreturn]] static void reportMissingDataLayout(Type type) {
27  std::string message;
28  llvm::raw_string_ostream os(message);
29  os << "neither the scoping op nor the type class provide data layout "
30  "information for "
31  << type;
32  llvm::report_fatal_error(Twine(message));
33 }
34 
35 /// Returns the bitwidth of the index type if specified in the param list.
36 /// Assumes 64-bit index otherwise.
37 static uint64_t getIndexBitwidth(DataLayoutEntryListRef params) {
38  if (params.empty())
39  return 64;
40  auto attr = cast<IntegerAttr>(params.front().getValue());
41  return attr.getValue().getZExtValue();
42 }
43 
44 llvm::TypeSize
47  llvm::TypeSize bits = getDefaultTypeSizeInBits(type, dataLayout, params);
48  return divideCeil(bits, 8);
49 }
50 
51 llvm::TypeSize
53  DataLayoutEntryListRef params) {
54  if (type.isIntOrFloat())
55  return llvm::TypeSize::getFixed(type.getIntOrFloatBitWidth());
56 
57  if (auto ctype = dyn_cast<ComplexType>(type)) {
58  Type et = ctype.getElementType();
59  uint64_t innerAlignment =
60  getDefaultPreferredAlignment(et, dataLayout, params) * 8;
61  llvm::TypeSize innerSize = getDefaultTypeSizeInBits(et, dataLayout, params);
62 
63  // Include padding required to align the imaginary value in the complex
64  // type.
65  return llvm::alignTo(innerSize, innerAlignment) + innerSize;
66  }
67 
68  // Index is an integer of some bitwidth.
69  if (isa<IndexType>(type))
70  return dataLayout.getTypeSizeInBits(
72 
73  // Sizes of vector types are rounded up to those of types with closest
74  // power-of-two number of elements in the innermost dimension. We also assume
75  // there is no bit-packing at the moment element sizes are taken in bytes and
76  // multiplied with 8 bits.
77  // TODO: make this extensible.
78  if (auto vecType = dyn_cast<VectorType>(type)) {
79  uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *
80  llvm::PowerOf2Ceil(vecType.getShape().back()) *
81  dataLayout.getTypeSize(vecType.getElementType()) * 8;
82  return llvm::TypeSize::get(baseSize, vecType.isScalable());
83  }
84 
85  if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
86  return typeInterface.getTypeSizeInBits(dataLayout, params);
87 
89 }
90 
91 static DataLayoutEntryInterface
92 findEntryForIntegerType(IntegerType intType,
94  assert(!params.empty() && "expected non-empty parameter list");
95  std::map<unsigned, DataLayoutEntryInterface> sortedParams;
96  for (DataLayoutEntryInterface entry : params) {
97  sortedParams.insert(std::make_pair(
98  cast<Type>(entry.getKey()).getIntOrFloatBitWidth(), entry));
99  }
100  auto iter = sortedParams.lower_bound(intType.getWidth());
101  if (iter == sortedParams.end())
102  iter = std::prev(iter);
103 
104  return iter->second;
105 }
106 
107 constexpr const static uint64_t kDefaultBitsInByte = 8u;
108 
109 static uint64_t extractABIAlignment(DataLayoutEntryInterface entry) {
110  auto values =
111  cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
112  return static_cast<uint64_t>(*values.begin()) / kDefaultBitsInByte;
113 }
114 
115 static uint64_t
116 getIntegerTypeABIAlignment(IntegerType intType,
118  constexpr uint64_t kDefaultSmallIntAlignment = 4u;
119  constexpr unsigned kSmallIntSize = 64;
120  if (params.empty()) {
121  return intType.getWidth() < kSmallIntSize
122  ? llvm::PowerOf2Ceil(
123  llvm::divideCeil(intType.getWidth(), kDefaultBitsInByte))
124  : kDefaultSmallIntAlignment;
125  }
126 
127  return extractABIAlignment(findEntryForIntegerType(intType, params));
128 }
129 
130 static uint64_t
131 getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout,
133  assert(params.size() <= 1 && "at most one data layout entry is expected for "
134  "the singleton floating-point type");
135  if (params.empty())
136  return llvm::PowerOf2Ceil(dataLayout.getTypeSize(fltType).getFixedValue());
137  return extractABIAlignment(params[0]);
138 }
139 
141  Type type, const DataLayout &dataLayout,
143  // Natural alignment is the closest power-of-two number above. For scalable
144  // vectors, aligning them to the same as the base vector is sufficient.
145  if (isa<VectorType>(type))
146  return llvm::PowerOf2Ceil(dataLayout.getTypeSize(type).getKnownMinValue());
147 
148  if (auto fltType = dyn_cast<FloatType>(type))
149  return getFloatTypeABIAlignment(fltType, dataLayout, params);
150 
151  // Index is an integer of some bitwidth.
152  if (isa<IndexType>(type))
153  return dataLayout.getTypeABIAlignment(
154  IntegerType::get(type.getContext(), getIndexBitwidth(params)));
155 
156  if (auto intType = dyn_cast<IntegerType>(type))
157  return getIntegerTypeABIAlignment(intType, params);
158 
159  if (auto ctype = dyn_cast<ComplexType>(type))
160  return getDefaultABIAlignment(ctype.getElementType(), dataLayout, params);
161 
162  if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
163  return typeInterface.getABIAlignment(dataLayout, params);
164 
166 }
167 
168 static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry) {
169  auto values =
170  cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
171  return *std::next(values.begin(), values.size() - 1) / kDefaultBitsInByte;
172 }
173 
174 static uint64_t
176  const DataLayout &dataLayout,
178  if (params.empty())
179  return llvm::PowerOf2Ceil(dataLayout.getTypeSize(intType).getFixedValue());
180 
181  return extractPreferredAlignment(findEntryForIntegerType(intType, params));
182 }
183 
184 static uint64_t
185 getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout,
187  assert(params.size() <= 1 && "at most one data layout entry is expected for "
188  "the singleton floating-point type");
189  if (params.empty())
190  return dataLayout.getTypeABIAlignment(fltType);
191  return extractPreferredAlignment(params[0]);
192 }
193 
195  Type type, const DataLayout &dataLayout,
197  // Preferred alignment is same as natural for floats and vectors.
198  if (isa<VectorType>(type))
199  return dataLayout.getTypeABIAlignment(type);
200 
201  if (auto fltType = dyn_cast<FloatType>(type))
202  return getFloatTypePreferredAlignment(fltType, dataLayout, params);
203 
204  // Preferred alignment is the closest power-of-two number above for integers
205  // (ABI alignment may be smaller).
206  if (auto intType = dyn_cast<IntegerType>(type))
207  return getIntegerTypePreferredAlignment(intType, dataLayout, params);
208 
209  if (isa<IndexType>(type)) {
210  return dataLayout.getTypePreferredAlignment(
211  IntegerType::get(type.getContext(), getIndexBitwidth(params)));
212  }
213 
214  if (auto ctype = dyn_cast<ComplexType>(type))
215  return getDefaultPreferredAlignment(ctype.getElementType(), dataLayout,
216  params);
217 
218  if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
219  return typeInterface.getPreferredAlignment(dataLayout, params);
220 
222 }
223 
225  Type type, const DataLayout &dataLayout,
227  if (isa<IndexType>(type))
228  return getIndexBitwidth(params);
229 
230  if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
231  if (std::optional<uint64_t> indexBitwidth =
232  typeInterface.getIndexBitwidth(dataLayout, params))
233  return *indexBitwidth;
234 
235  // Return std::nullopt for all other types, which are assumed to be non
236  // pointer-like types.
237  return std::nullopt;
238 }
239 
240 // Returns the endianness if specified in the given entry. If the entry is empty
241 // the default endianness represented by an empty attribute is returned.
242 Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) {
243  if (entry == DataLayoutEntryInterface())
244  return Attribute();
245 
246  return entry.getValue();
247 }
248 
249 // Returns the default memory space if specified in the given entry. If the
250 // entry is empty the default memory space represented by an empty attribute is
251 // returned.
252 Attribute mlir::detail::getDefaultMemorySpace(DataLayoutEntryInterface entry) {
253  if (!entry)
254  return Attribute();
255 
256  return entry.getValue();
257 }
258 
259 // Returns the memory space used for alloca operations if specified in the
260 // given entry. If the entry is empty the default memory space represented by
261 // an empty attribute is returned.
262 Attribute
263 mlir::detail::getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry) {
264  if (entry == DataLayoutEntryInterface()) {
265  return Attribute();
266  }
267 
268  return entry.getValue();
269 }
270 
271 // Returns the mangling mode if specified in the given entry.
272 // If the entry is empty, an empty attribute is returned.
273 Attribute mlir::detail::getDefaultManglingMode(DataLayoutEntryInterface entry) {
274  if (entry == DataLayoutEntryInterface())
275  return Attribute();
276 
277  return entry.getValue();
278 }
279 
280 // Returns the memory space used for the program memory space. if
281 // specified in the given entry. If the entry is empty the default
282 // memory space represented by an empty attribute is returned.
283 Attribute
284 mlir::detail::getDefaultProgramMemorySpace(DataLayoutEntryInterface entry) {
285  if (entry == DataLayoutEntryInterface()) {
286  return Attribute();
287  }
288 
289  return entry.getValue();
290 }
291 
292 // Returns the memory space used for global the global memory space. if
293 // specified in the given entry. If the entry is empty the default memory
294 // space represented by an empty attribute is returned.
295 Attribute
296 mlir::detail::getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry) {
297  if (entry == DataLayoutEntryInterface()) {
298  return Attribute();
299  }
300 
301  return entry.getValue();
302 }
303 
304 // Returns the stack alignment if specified in the given entry. If the entry is
305 // empty the default alignment zero is returned.
306 uint64_t
307 mlir::detail::getDefaultStackAlignment(DataLayoutEntryInterface entry) {
308  if (entry == DataLayoutEntryInterface())
309  return 0;
310 
311  auto value = cast<IntegerAttr>(entry.getValue());
312  return value.getValue().getZExtValue();
313 }
314 
315 std::optional<Attribute>
316 mlir::detail::getDevicePropertyValue(DataLayoutEntryInterface entry) {
317  if (entry == DataLayoutEntryInterface())
318  return std::nullopt;
319 
320  return entry.getValue();
321 }
322 
325  TypeID typeID) {
326  return llvm::filter_to_vector<4>(
327  entries, [typeID](DataLayoutEntryInterface entry) {
328  auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
329  return type && type.getTypeID() == typeID;
330  });
331 }
332 
333 DataLayoutEntryInterface
335  StringAttr id) {
336  const auto *it = llvm::find_if(entries, [id](DataLayoutEntryInterface entry) {
337  if (auto attr = dyn_cast<StringAttr>(entry.getKey()))
338  return attr == id;
339  return false;
340  });
341  return it == entries.end() ? DataLayoutEntryInterface() : *it;
342 }
343 
344 static DataLayoutSpecInterface getSpec(Operation *operation) {
346  .Case<ModuleOp, DataLayoutOpInterface>(
347  [&](auto op) { return op.getDataLayoutSpec(); })
348  .Default([](Operation *) {
349  llvm_unreachable("expected an op with data layout spec");
350  return DataLayoutSpecInterface();
351  });
352 }
353 
354 static TargetSystemSpecInterface getTargetSystemSpec(Operation *operation) {
355  if (operation) {
356  ModuleOp moduleOp = dyn_cast<ModuleOp>(operation);
357  if (!moduleOp)
358  moduleOp = operation->getParentOfType<ModuleOp>();
359  return moduleOp.getTargetSystemSpec();
360  }
361  return TargetSystemSpecInterface();
362 }
363 
364 /// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that
365 /// are either modules or implement the `DataLayoutOpInterface`.
366 static void
369  SmallVectorImpl<Location> *opLocations = nullptr) {
370  if (!leaf)
371  return;
372 
373  for (Operation *parent = leaf->getParentOp(); parent != nullptr;
374  parent = parent->getParentOp()) {
376  .Case<ModuleOp>([&](ModuleOp op) {
377  // Skip top-level module op unless it has a layout. Top-level module
378  // without layout is most likely the one implicitly added by the
379  // parser and it doesn't have location. Top-level null specification
380  // would have had the same effect as not having a specification at all
381  // (using type defaults).
382  if (!op->getParentOp() && !op.getDataLayoutSpec())
383  return;
384  specs.push_back(op.getDataLayoutSpec());
385  if (opLocations)
386  opLocations->push_back(op.getLoc());
387  })
388  .Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {
389  specs.push_back(op.getDataLayoutSpec());
390  if (opLocations)
391  opLocations->push_back(op.getLoc());
392  });
393  }
394 }
395 
396 /// Returns a layout spec that is a combination of the layout specs attached
397 /// to the given operation and all its ancestors.
398 static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {
399  if (!leaf)
400  return {};
401 
402  assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
403  "expected an op with data layout spec");
404 
406  collectParentLayouts(leaf, specs);
407 
408  // Fast track if there are no ancestors.
409  if (specs.empty())
410  return getSpec(leaf);
411 
412  // Create the list of non-null specs (null/missing specs can be safely
413  // ignored) from the outermost to the innermost.
414  auto nonNullSpecs = llvm::filter_to_vector<2>(
415  llvm::reverse(specs),
416  [](DataLayoutSpecInterface iface) { return iface != nullptr; });
417 
418  // Combine the specs using the innermost as anchor.
419  if (DataLayoutSpecInterface current = getSpec(leaf))
420  return current.combineWith(nonNullSpecs);
421  if (nonNullSpecs.empty())
422  return {};
423  return nonNullSpecs.back().combineWith(
424  llvm::ArrayRef(nonNullSpecs).drop_back());
425 }
426 
428  DataLayoutSpecInterface spec = getSpec(op);
429  // The layout specification may be missing and it's fine.
430  if (!spec)
431  return success();
432 
433  if (failed(spec.verifySpec(op->getLoc())))
434  return failure();
435  if (!getCombinedDataLayout(op)) {
437  op->emitError()
438  << "data layout does not combine with layouts of enclosing ops";
440  SmallVector<Location> opLocations;
441  collectParentLayouts(op, specs, &opLocations);
442  for (Location loc : opLocations)
443  diag.attachNote(loc) << "enclosing op with data layout";
444  return diag;
445  }
446  return success();
447 }
448 
449 llvm::TypeSize mlir::detail::divideCeil(llvm::TypeSize numerator,
450  uint64_t denominator) {
451  uint64_t divided =
452  llvm::divideCeil(numerator.getKnownMinValue(), denominator);
453  return llvm::TypeSize::get(divided, numerator.isScalable());
454 }
455 
456 //===----------------------------------------------------------------------===//
457 // DataLayout
458 //===----------------------------------------------------------------------===//
459 
460 template <typename OpTy>
461 void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {
462  if (!originalLayout) {
463  assert((!op || !op.getDataLayoutSpec()) &&
464  "could not compute layout information for an op (failed to "
465  "combine attributes?)");
466  }
467 }
468 
470 
471 mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
472  : originalLayout(getCombinedDataLayout(op)),
473  originalTargetSystemDesc(getTargetSystemSpec(op)), scope(op),
474  allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
475  globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
476 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
477  checkMissingLayout(originalLayout, op);
478  collectParentLayouts(op, layoutStack);
479 #endif
480 }
481 
483  : originalLayout(getCombinedDataLayout(op)),
484  originalTargetSystemDesc(getTargetSystemSpec(op)), scope(op),
485  allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
486  globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
487 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
488  checkMissingLayout(originalLayout, op);
489  collectParentLayouts(op, layoutStack);
490 #endif
491 }
492 
494  // Search the closest parent either being a module operation or implementing
495  // the data layout interface.
496  while (op) {
497  if (auto module = dyn_cast<ModuleOp>(op))
498  return DataLayout(module);
499  if (auto iface = dyn_cast<DataLayoutOpInterface>(op))
500  return DataLayout(iface);
501  op = op->getParentOp();
502  }
503  return DataLayout();
504 }
505 
506 void mlir::DataLayout::checkValid() const {
507 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
509  collectParentLayouts(scope, specs);
510  assert(specs.size() == layoutStack.size() &&
511  "data layout object used, but no longer valid due to the change in "
512  "number of nested layouts");
513  for (auto pair : llvm::zip(specs, layoutStack)) {
514  Attribute newLayout = std::get<0>(pair);
515  Attribute origLayout = std::get<1>(pair);
516  assert(newLayout == origLayout &&
517  "data layout object used, but no longer valid "
518  "due to the change in layout attributes");
519  }
520 #endif
521  assert(((!scope && !this->originalLayout) ||
522  (scope && this->originalLayout == getCombinedDataLayout(scope))) &&
523  "data layout object used, but no longer valid due to the change in "
524  "layout spec");
525 }
526 
527 /// Looks up the value for the given type key in the given cache. If there is no
528 /// such value in the cache, compute it using the given callback and put it in
529 /// the cache before returning.
530 template <typename T>
532  function_ref<T(Type)> compute) {
533  auto it = cache.find(t);
534  if (it != cache.end())
535  return it->second;
536 
537  auto result = cache.try_emplace(t, compute(t));
538  return result.first->second;
539 }
540 
541 llvm::TypeSize mlir::DataLayout::getTypeSize(Type t) const {
542  checkValid();
543  return cachedLookup<llvm::TypeSize>(t, sizes, [&](Type ty) {
544  DataLayoutEntryList list;
545  if (originalLayout)
546  list = originalLayout.getSpecForType(ty.getTypeID());
547  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
548  return iface.getTypeSize(ty, *this, list);
549  return detail::getDefaultTypeSize(ty, *this, list);
550  });
551 }
552 
553 llvm::TypeSize mlir::DataLayout::getTypeSizeInBits(Type t) const {
554  checkValid();
555  return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](Type ty) {
556  DataLayoutEntryList list;
557  if (originalLayout)
558  list = originalLayout.getSpecForType(ty.getTypeID());
559  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
560  return iface.getTypeSizeInBits(ty, *this, list);
561  return detail::getDefaultTypeSizeInBits(ty, *this, list);
562  });
563 }
564 
566  checkValid();
567  return cachedLookup<uint64_t>(t, abiAlignments, [&](Type ty) {
568  DataLayoutEntryList list;
569  if (originalLayout)
570  list = originalLayout.getSpecForType(ty.getTypeID());
571  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
572  return iface.getTypeABIAlignment(ty, *this, list);
573  return detail::getDefaultABIAlignment(ty, *this, list);
574  });
575 }
576 
578  checkValid();
579  return cachedLookup<uint64_t>(t, preferredAlignments, [&](Type ty) {
580  DataLayoutEntryList list;
581  if (originalLayout)
582  list = originalLayout.getSpecForType(ty.getTypeID());
583  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
584  return iface.getTypePreferredAlignment(ty, *this, list);
585  return detail::getDefaultPreferredAlignment(ty, *this, list);
586  });
587 }
588 
589 std::optional<uint64_t> mlir::DataLayout::getTypeIndexBitwidth(Type t) const {
590  checkValid();
591  return cachedLookup<std::optional<uint64_t>>(t, indexBitwidths, [&](Type ty) {
592  DataLayoutEntryList list;
593  if (originalLayout)
594  list = originalLayout.getSpecForType(ty.getTypeID());
595  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
596  return iface.getIndexBitwidth(ty, *this, list);
597  return detail::getDefaultIndexBitwidth(ty, *this, list);
598  });
599 }
600 
602  checkValid();
603  if (endianness)
604  return *endianness;
605  DataLayoutEntryInterface entry;
606  if (originalLayout)
607  entry = originalLayout.getSpecForIdentifier(
608  originalLayout.getEndiannessIdentifier(originalLayout.getContext()));
609 
610  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
611  endianness = iface.getEndianness(entry);
612  else
613  endianness = detail::getDefaultEndianness(entry);
614  return *endianness;
615 }
616 
618  checkValid();
619  if (defaultMemorySpace)
620  return *defaultMemorySpace;
621  DataLayoutEntryInterface entry;
622  if (originalLayout)
623  entry = originalLayout.getSpecForIdentifier(
624  originalLayout.getDefaultMemorySpaceIdentifier(
625  originalLayout.getContext()));
626  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
627  defaultMemorySpace = iface.getDefaultMemorySpace(entry);
628  else
629  defaultMemorySpace = detail::getDefaultMemorySpace(entry);
630  return *defaultMemorySpace;
631 }
632 
634  checkValid();
635  if (allocaMemorySpace)
636  return *allocaMemorySpace;
637  DataLayoutEntryInterface entry;
638  if (originalLayout)
639  entry = originalLayout.getSpecForIdentifier(
640  originalLayout.getAllocaMemorySpaceIdentifier(
641  originalLayout.getContext()));
642  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
643  allocaMemorySpace = iface.getAllocaMemorySpace(entry);
644  else
645  allocaMemorySpace = detail::getDefaultAllocaMemorySpace(entry);
646  return *allocaMemorySpace;
647 }
648 
650  checkValid();
651  if (manglingMode)
652  return *manglingMode;
653  DataLayoutEntryInterface entry;
654  if (originalLayout)
655  entry = originalLayout.getSpecForIdentifier(
656  originalLayout.getManglingModeIdentifier(originalLayout.getContext()));
657 
658  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
659  manglingMode = iface.getManglingMode(entry);
660  else
661  manglingMode = detail::getDefaultManglingMode(entry);
662  return *manglingMode;
663 }
664 
666  checkValid();
667  if (programMemorySpace)
668  return *programMemorySpace;
669  DataLayoutEntryInterface entry;
670  if (originalLayout)
671  entry = originalLayout.getSpecForIdentifier(
672  originalLayout.getProgramMemorySpaceIdentifier(
673  originalLayout.getContext()));
674  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
675  programMemorySpace = iface.getProgramMemorySpace(entry);
676  else
677  programMemorySpace = detail::getDefaultProgramMemorySpace(entry);
678  return *programMemorySpace;
679 }
680 
682  checkValid();
683  if (globalMemorySpace)
684  return *globalMemorySpace;
685  DataLayoutEntryInterface entry;
686  if (originalLayout)
687  entry = originalLayout.getSpecForIdentifier(
688  originalLayout.getGlobalMemorySpaceIdentifier(
689  originalLayout.getContext()));
690  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
691  globalMemorySpace = iface.getGlobalMemorySpace(entry);
692  else
693  globalMemorySpace = detail::getDefaultGlobalMemorySpace(entry);
694  return *globalMemorySpace;
695 }
696 
698  checkValid();
699  if (stackAlignment)
700  return *stackAlignment;
701  DataLayoutEntryInterface entry;
702  if (originalLayout)
703  entry = originalLayout.getSpecForIdentifier(
704  originalLayout.getStackAlignmentIdentifier(
705  originalLayout.getContext()));
706  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
707  stackAlignment = iface.getStackAlignment(entry);
708  else
709  stackAlignment = detail::getDefaultStackAlignment(entry);
710  return *stackAlignment;
711 }
712 
714  TargetSystemSpecInterface::DeviceID deviceID,
715  StringAttr propertyName) const {
716  checkValid();
717  DataLayoutEntryInterface entry;
718  if (originalTargetSystemDesc) {
719  if (std::optional<TargetDeviceSpecInterface> device =
720  originalTargetSystemDesc.getDeviceSpecForDeviceID(deviceID))
721  entry = device->getSpecForIdentifier(propertyName);
722  }
723  // Currently I am not caching the results because we do not return
724  // default values of these properties. Instead if the property is
725  // missing, we return std::nullopt so that the users can resort to
726  // the default value however they want.
727  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
728  return iface.getDevicePropertyValue(entry);
729  else
730  return detail::getDevicePropertyValue(entry);
731 }
732 
733 //===----------------------------------------------------------------------===//
734 // DataLayoutSpecInterface
735 //===----------------------------------------------------------------------===//
736 
737 void DataLayoutSpecInterface::bucketEntriesByType(
738  llvm::MapVector<TypeID, DataLayoutEntryList> &types,
739  llvm::MapVector<StringAttr, DataLayoutEntryInterface> &ids) {
740  for (DataLayoutEntryInterface entry : getEntries()) {
741  if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))
742  types[type.getTypeID()].push_back(entry);
743  else
744  ids[llvm::cast<StringAttr>(entry.getKey())] = entry;
745  }
746 }
747 
748 LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,
749  Location loc) {
750  // First, verify individual entries.
751  for (DataLayoutEntryInterface entry : spec.getEntries())
752  if (failed(entry.verifyEntry(loc)))
753  return failure();
754 
755  // Second, dispatch verifications of entry groups to types or dialects they
756  // are associated with.
757  llvm::MapVector<TypeID, DataLayoutEntryList> types;
758  llvm::MapVector<StringAttr, DataLayoutEntryInterface> ids;
759  spec.bucketEntriesByType(types, ids);
760 
761  for (const auto &kvp : types) {
762  auto sampleType = cast<Type>(kvp.second.front().getKey());
763  if (isa<IndexType>(sampleType)) {
764  assert(kvp.second.size() == 1 &&
765  "expected one data layout entry for non-parametric 'index' type");
766  if (!isa<IntegerAttr>(kvp.second.front().getValue()))
767  return emitError(loc)
768  << "expected integer attribute in the data layout entry for "
769  << sampleType;
770  continue;
771  }
772 
773  if (sampleType.isIntOrFloat()) {
774  for (DataLayoutEntryInterface entry : kvp.second) {
775  auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());
776  if (!value || !value.getElementType().isSignlessInteger(64)) {
777  emitError(loc) << "expected a dense i64 elements attribute in the "
778  "data layout entry "
779  << entry;
780  return failure();
781  }
782 
783  auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());
784  unsigned numElements = elements.size();
785  if (numElements < 1 || numElements > 2) {
786  emitError(loc) << "expected 1 or 2 elements in the data layout entry "
787  << entry;
788  return failure();
789  }
790 
791  uint64_t abi = elements[0];
792  uint64_t preferred = numElements == 2 ? elements[1] : abi;
793  if (preferred < abi) {
794  emitError(loc)
795  << "preferred alignment is expected to be greater than or equal "
796  "to the abi alignment in data layout entry "
797  << entry;
798  return failure();
799  }
800  }
801  continue;
802  }
803 
804  if (isa<BuiltinDialect>(&sampleType.getDialect()))
805  return emitError(loc) << "unexpected data layout for a built-in type";
806 
807  auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);
808  if (!dlType)
809  return emitError(loc)
810  << "data layout specified for a type that does not support it";
811  if (failed(dlType.verifyEntries(kvp.second, loc)))
812  return failure();
813  }
814 
815  for (const auto &kvp : ids) {
816  StringAttr identifier = cast<StringAttr>(kvp.second.getKey());
817  Dialect *dialect = identifier.getReferencedDialect();
818 
819  // Ignore attributes that belong to an unknown dialect, the dialect may
820  // actually implement the relevant interface but we don't know about that.
821  if (!dialect)
822  continue;
823 
824  const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
825  if (!iface) {
826  return emitError(loc)
827  << "the '" << dialect->getNamespace()
828  << "' dialect does not support identifier data layout entries";
829  }
830  if (failed(iface->verifyEntry(kvp.second, loc)))
831  return failure();
832  }
833 
834  return success();
835 }
836 
837 LogicalResult
838 mlir::detail::verifyTargetSystemSpec(TargetSystemSpecInterface spec,
839  Location loc) {
842  for (const auto &entry : spec.getEntries()) {
843  auto targetDeviceSpec =
844  dyn_cast<TargetDeviceSpecInterface>(entry.getValue());
845 
846  if (!targetDeviceSpec)
847  return failure();
848 
849  // First, verify individual target device desc specs.
850  if (failed(targetDeviceSpec.verifyEntry(loc)))
851  return failure();
852 
853  // Check that device IDs are unique across all entries.
854  auto deviceID =
855  llvm::dyn_cast<TargetSystemSpecInterface::DeviceID>(entry.getKey());
856  if (!deviceID)
857  return failure();
858 
859  if (!deviceIDs.insert(deviceID).second) {
860  return failure();
861  }
862 
863  // collect all the keys used by all the target device specs.
864  for (DataLayoutEntryInterface entry : targetDeviceSpec.getEntries()) {
865  if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey())) {
866  // targetDeviceSpec does not support Type as a key.
867  return failure();
868  } else {
869  deviceDescKeys[cast<StringAttr>(entry.getKey())] = entry;
870  }
871  }
872  }
873 
874  for (const auto &[keyName, keyVal] : deviceDescKeys) {
875  Dialect *dialect = keyName.getReferencedDialect();
876 
877  // Ignore attributes that belong to an unknown dialect, the dialect may
878  // actually implement the relevant interface but we don't know about that.
879  if (!dialect)
880  return failure();
881 
882  const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
883  if (!iface) {
884  return emitError(loc)
885  << "the '" << dialect->getNamespace()
886  << "' dialect does not support identifier data layout entries";
887  }
888  if (failed(iface->verifyEntry(keyVal, loc)))
889  return failure();
890  }
891 
892  return success();
893 }
894 
895 #include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
896 #include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
897 #include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"
static uint64_t getIntegerTypePreferredAlignment(IntegerType intType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf)
Returns a layout spec that is a combination of the layout specs attached to the given operation and a...
static TargetSystemSpecInterface getTargetSystemSpec(Operation *operation)
constexpr static const uint64_t kDefaultBitsInByte
static uint64_t getIntegerTypeABIAlignment(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
static T cachedLookup(Type t, DenseMap< Type, T > &cache, function_ref< T(Type)> compute)
Looks up the value for the given type key in the given cache.
static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry)
static void collectParentLayouts(Operation *leaf, SmallVectorImpl< DataLayoutSpecInterface > &specs, SmallVectorImpl< Location > *opLocations=nullptr)
Populates opsWithLayout with the list of proper ancestors of leaf that are either modules or implemen...
static uint64_t getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static uint64_t getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getSpec(Operation *operation)
static uint64_t getIndexBitwidth(DataLayoutEntryListRef params)
Returns the bitwidth of the index type if specified in the param list.
static uint64_t extractABIAlignment(DataLayoutEntryInterface entry)
static DataLayoutEntryInterface findEntryForIntegerType(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op)
static void reportMissingDataLayout(Type type)
Reports that the given type is missing the data layout information and exits.
static std::string diag(const llvm::Value &value)
Attributes are known-constant values of operations.
Definition: Attributes.h:25
The main mechanism for performing data layout queries.
Attribute getAllocaMemorySpace() const
Returns the memory space used for AllocaOps.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
std::optional< uint64_t > getTypeIndexBitwidth(Type t) const
Returns the bitwidth that should be used when performing index computations for the given pointer-lik...
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
Attribute getManglingMode() const
Returns the mangling mode.
uint64_t getStackAlignment() const
Returns the natural alignment of the stack in bits.
Attribute getProgramMemorySpace() const
Returns the memory space used for program memory operations.
uint64_t getTypePreferredAlignment(Type t) const
Returns the preferred of the given type in the current scope.
Attribute getGlobalMemorySpace() const
Returns the memory space used for global operations.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Attribute getDefaultMemorySpace() const
Returns the default memory space used for memory operations.
Attribute getEndianness() const
Returns the specified endianness.
std::optional< Attribute > getDevicePropertyValue(TargetSystemSpecInterface::DeviceID, StringAttr propertyName) const
Returns the value of the specified property if the property is defined for the given device ID,...
Dialects are groups of MLIR operations, types and attributes, as well as behavior associated with the...
Definition: Dialect.h:38
StringRef getNamespace() const
Definition: Dialect.h:54
This class represents a diagnostic that is inflight and set to be reported.
Definition: Diagnostics.h:314
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:66
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition: Operation.h:234
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:268
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
Definition: Operation.h:238
This class provides an efficient unique identifier for a specific C++ type.
Definition: TypeID.h:107
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
Definition: Types.cpp:35
bool isIntOrFloat() const
Return true if this is an integer (of any signedness) or a float type.
Definition: Types.cpp:116
TypeID getTypeID()
Return a unique identifier for the concrete type.
Definition: Types.h:101
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition: Types.cpp:122
Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry)
Default handler for alloca memory space request.
Attribute getDefaultProgramMemorySpace(DataLayoutEntryInterface entry)
Default handler for program memory space request.
Attribute getDefaultEndianness(DataLayoutEntryInterface entry)
Default handler for endianness request.
std::optional< uint64_t > getDefaultIndexBitwidth(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the index bitwidth request.
DataLayoutEntryList filterEntriesForType(DataLayoutEntryListRef entries, TypeID typeID)
Given a list of data layout entries, returns a new list containing the entries with keys having the g...
LogicalResult verifyTargetSystemSpec(TargetSystemSpecInterface spec, Location loc)
Verifies that a target system desc spec is valid.
std::optional< Attribute > getDevicePropertyValue(DataLayoutEntryInterface entry)
Returns the value of the property from the specified DataLayoutEntry.
Attribute getDefaultManglingMode(DataLayoutEntryInterface entry)
Default handler for mangling mode request.
uint64_t getDefaultABIAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the required alignment request.
llvm::TypeSize getDefaultTypeSize(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size request.
llvm::TypeSize getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size in bits request.
uint64_t getDefaultPreferredAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the preferred alignment request.
llvm::TypeSize divideCeil(llvm::TypeSize numerator, uint64_t denominator)
Divides the known min value of the numerator by the denominator and rounds the result up to the next ...
uint64_t getDefaultStackAlignment(DataLayoutEntryInterface entry)
Default handler for the stack alignment request.
Attribute getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry)
Default handler for global memory space request.
Attribute getDefaultMemorySpace(DataLayoutEntryInterface entry)
Default handler for the default memory space request.
LogicalResult verifyDataLayoutOp(Operation *op)
Verifies that the operation implementing the data layout interface, or a module operation,...
LogicalResult verifyDataLayoutSpec(DataLayoutSpecInterface spec, Location loc)
Verifies that a data layout spec is valid.
DataLayoutEntryInterface filterEntryForIdentifier(DataLayoutEntryListRef entries, StringAttr id)
Given a list of data layout entries, returns the entry that has the given identifier as key,...
Include the generated interface declarations.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...