MLIR  21.0.0git
DataLayoutInterfaces.cpp
Go to the documentation of this file.
1 //===- DataLayoutInterfaces.cpp - Data Layout Interface Implementation ----===//
2 //
3 // Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4 // See https://llvm.org/LICENSE.txt for license information.
5 // SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6 //
7 //===----------------------------------------------------------------------===//
8 
10 #include "mlir/IR/BuiltinDialect.h"
11 #include "mlir/IR/BuiltinOps.h"
12 #include "mlir/IR/BuiltinTypes.h"
13 #include "mlir/IR/Operation.h"
14 
15 #include "llvm/ADT/TypeSwitch.h"
16 #include "llvm/Support/MathExtras.h"
17 
18 using namespace mlir;
19 
20 //===----------------------------------------------------------------------===//
21 // Default implementations
22 //===----------------------------------------------------------------------===//
23 
24 /// Reports that the given type is missing the data layout information and
25 /// exits.
26 [[noreturn]] static void reportMissingDataLayout(Type type) {
27  std::string message;
28  llvm::raw_string_ostream os(message);
29  os << "neither the scoping op nor the type class provide data layout "
30  "information for "
31  << type;
32  llvm::report_fatal_error(Twine(message));
33 }
34 
35 /// Returns the bitwidth of the index type if specified in the param list.
36 /// Assumes 64-bit index otherwise.
37 static uint64_t getIndexBitwidth(DataLayoutEntryListRef params) {
38  if (params.empty())
39  return 64;
40  auto attr = cast<IntegerAttr>(params.front().getValue());
41  return attr.getValue().getZExtValue();
42 }
43 
44 llvm::TypeSize
47  llvm::TypeSize bits = getDefaultTypeSizeInBits(type, dataLayout, params);
48  return divideCeil(bits, 8);
49 }
50 
51 llvm::TypeSize
53  DataLayoutEntryListRef params) {
54  if (type.isIntOrFloat())
55  return llvm::TypeSize::getFixed(type.getIntOrFloatBitWidth());
56 
57  if (auto ctype = dyn_cast<ComplexType>(type)) {
58  Type et = ctype.getElementType();
59  uint64_t innerAlignment =
60  getDefaultPreferredAlignment(et, dataLayout, params) * 8;
61  llvm::TypeSize innerSize = getDefaultTypeSizeInBits(et, dataLayout, params);
62 
63  // Include padding required to align the imaginary value in the complex
64  // type.
65  return llvm::alignTo(innerSize, innerAlignment) + innerSize;
66  }
67 
68  // Index is an integer of some bitwidth.
69  if (isa<IndexType>(type))
70  return dataLayout.getTypeSizeInBits(
72 
73  // Sizes of vector types are rounded up to those of types with closest
74  // power-of-two number of elements in the innermost dimension. We also assume
75  // there is no bit-packing at the moment element sizes are taken in bytes and
76  // multiplied with 8 bits.
77  // TODO: make this extensible.
78  if (auto vecType = dyn_cast<VectorType>(type)) {
79  uint64_t baseSize = vecType.getNumElements() / vecType.getShape().back() *
80  llvm::PowerOf2Ceil(vecType.getShape().back()) *
81  dataLayout.getTypeSize(vecType.getElementType()) * 8;
82  return llvm::TypeSize::get(baseSize, vecType.isScalable());
83  }
84 
85  if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
86  return typeInterface.getTypeSizeInBits(dataLayout, params);
87 
89 }
90 
91 static DataLayoutEntryInterface
92 findEntryForIntegerType(IntegerType intType,
94  assert(!params.empty() && "expected non-empty parameter list");
95  std::map<unsigned, DataLayoutEntryInterface> sortedParams;
96  for (DataLayoutEntryInterface entry : params) {
97  sortedParams.insert(std::make_pair(
98  cast<Type>(entry.getKey()).getIntOrFloatBitWidth(), entry));
99  }
100  auto iter = sortedParams.lower_bound(intType.getWidth());
101  if (iter == sortedParams.end())
102  iter = std::prev(iter);
103 
104  return iter->second;
105 }
106 
107 constexpr const static uint64_t kDefaultBitsInByte = 8u;
108 
109 static uint64_t extractABIAlignment(DataLayoutEntryInterface entry) {
110  auto values =
111  cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
112  return static_cast<uint64_t>(*values.begin()) / kDefaultBitsInByte;
113 }
114 
115 static uint64_t
116 getIntegerTypeABIAlignment(IntegerType intType,
118  constexpr uint64_t kDefaultSmallIntAlignment = 4u;
119  constexpr unsigned kSmallIntSize = 64;
120  if (params.empty()) {
121  return intType.getWidth() < kSmallIntSize
122  ? llvm::PowerOf2Ceil(
123  llvm::divideCeil(intType.getWidth(), kDefaultBitsInByte))
124  : kDefaultSmallIntAlignment;
125  }
126 
127  return extractABIAlignment(findEntryForIntegerType(intType, params));
128 }
129 
130 static uint64_t
131 getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout,
133  assert(params.size() <= 1 && "at most one data layout entry is expected for "
134  "the singleton floating-point type");
135  if (params.empty())
136  return llvm::PowerOf2Ceil(dataLayout.getTypeSize(fltType).getFixedValue());
137  return extractABIAlignment(params[0]);
138 }
139 
141  Type type, const DataLayout &dataLayout,
143  // Natural alignment is the closest power-of-two number above. For scalable
144  // vectors, aligning them to the same as the base vector is sufficient.
145  if (isa<VectorType>(type))
146  return llvm::PowerOf2Ceil(dataLayout.getTypeSize(type).getKnownMinValue());
147 
148  if (auto fltType = dyn_cast<FloatType>(type))
149  return getFloatTypeABIAlignment(fltType, dataLayout, params);
150 
151  // Index is an integer of some bitwidth.
152  if (isa<IndexType>(type))
153  return dataLayout.getTypeABIAlignment(
154  IntegerType::get(type.getContext(), getIndexBitwidth(params)));
155 
156  if (auto intType = dyn_cast<IntegerType>(type))
157  return getIntegerTypeABIAlignment(intType, params);
158 
159  if (auto ctype = dyn_cast<ComplexType>(type))
160  return getDefaultABIAlignment(ctype.getElementType(), dataLayout, params);
161 
162  if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
163  return typeInterface.getABIAlignment(dataLayout, params);
164 
166 }
167 
168 static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry) {
169  auto values =
170  cast<DenseIntElementsAttr>(entry.getValue()).getValues<uint64_t>();
171  return *std::next(values.begin(), values.size() - 1) / kDefaultBitsInByte;
172 }
173 
174 static uint64_t
176  const DataLayout &dataLayout,
178  if (params.empty())
179  return llvm::PowerOf2Ceil(dataLayout.getTypeSize(intType).getFixedValue());
180 
181  return extractPreferredAlignment(findEntryForIntegerType(intType, params));
182 }
183 
184 static uint64_t
185 getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout,
187  assert(params.size() <= 1 && "at most one data layout entry is expected for "
188  "the singleton floating-point type");
189  if (params.empty())
190  return dataLayout.getTypeABIAlignment(fltType);
191  return extractPreferredAlignment(params[0]);
192 }
193 
195  Type type, const DataLayout &dataLayout,
197  // Preferred alignment is same as natural for floats and vectors.
198  if (isa<VectorType>(type))
199  return dataLayout.getTypeABIAlignment(type);
200 
201  if (auto fltType = dyn_cast<FloatType>(type))
202  return getFloatTypePreferredAlignment(fltType, dataLayout, params);
203 
204  // Preferred alignment is the closest power-of-two number above for integers
205  // (ABI alignment may be smaller).
206  if (auto intType = dyn_cast<IntegerType>(type))
207  return getIntegerTypePreferredAlignment(intType, dataLayout, params);
208 
209  if (isa<IndexType>(type)) {
210  return dataLayout.getTypePreferredAlignment(
211  IntegerType::get(type.getContext(), getIndexBitwidth(params)));
212  }
213 
214  if (auto ctype = dyn_cast<ComplexType>(type))
215  return getDefaultPreferredAlignment(ctype.getElementType(), dataLayout,
216  params);
217 
218  if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
219  return typeInterface.getPreferredAlignment(dataLayout, params);
220 
222 }
223 
225  Type type, const DataLayout &dataLayout,
227  if (isa<IndexType>(type))
228  return getIndexBitwidth(params);
229 
230  if (auto typeInterface = dyn_cast<DataLayoutTypeInterface>(type))
231  if (std::optional<uint64_t> indexBitwidth =
232  typeInterface.getIndexBitwidth(dataLayout, params))
233  return *indexBitwidth;
234 
235  // Return std::nullopt for all other types, which are assumed to be non
236  // pointer-like types.
237  return std::nullopt;
238 }
239 
240 // Returns the endianness if specified in the given entry. If the entry is empty
241 // the default endianness represented by an empty attribute is returned.
242 Attribute mlir::detail::getDefaultEndianness(DataLayoutEntryInterface entry) {
243  if (entry == DataLayoutEntryInterface())
244  return Attribute();
245 
246  return entry.getValue();
247 }
248 
249 // Returns the default memory space if specified in the given entry. If the
250 // entry is empty the default memory space represented by an empty attribute is
251 // returned.
252 Attribute mlir::detail::getDefaultMemorySpace(DataLayoutEntryInterface entry) {
253  if (!entry)
254  return Attribute();
255 
256  return entry.getValue();
257 }
258 
259 // Returns the memory space used for alloca operations if specified in the
260 // given entry. If the entry is empty the default memory space represented by
261 // an empty attribute is returned.
262 Attribute
263 mlir::detail::getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry) {
264  if (entry == DataLayoutEntryInterface()) {
265  return Attribute();
266  }
267 
268  return entry.getValue();
269 }
270 
271 // Returns the mangling mode if specified in the given entry.
272 // If the entry is empty, an empty attribute is returned.
273 Attribute mlir::detail::getDefaultManglingMode(DataLayoutEntryInterface entry) {
274  if (entry == DataLayoutEntryInterface())
275  return Attribute();
276 
277  return entry.getValue();
278 }
279 
280 // Returns the memory space used for the program memory space. if
281 // specified in the given entry. If the entry is empty the default
282 // memory space represented by an empty attribute is returned.
283 Attribute
284 mlir::detail::getDefaultProgramMemorySpace(DataLayoutEntryInterface entry) {
285  if (entry == DataLayoutEntryInterface()) {
286  return Attribute();
287  }
288 
289  return entry.getValue();
290 }
291 
292 // Returns the memory space used for global the global memory space. if
293 // specified in the given entry. If the entry is empty the default memory
294 // space represented by an empty attribute is returned.
295 Attribute
296 mlir::detail::getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry) {
297  if (entry == DataLayoutEntryInterface()) {
298  return Attribute();
299  }
300 
301  return entry.getValue();
302 }
303 
304 // Returns the stack alignment if specified in the given entry. If the entry is
305 // empty the default alignment zero is returned.
306 uint64_t
307 mlir::detail::getDefaultStackAlignment(DataLayoutEntryInterface entry) {
308  if (entry == DataLayoutEntryInterface())
309  return 0;
310 
311  auto value = cast<IntegerAttr>(entry.getValue());
312  return value.getValue().getZExtValue();
313 }
314 
315 std::optional<Attribute>
316 mlir::detail::getDevicePropertyValue(DataLayoutEntryInterface entry) {
317  if (entry == DataLayoutEntryInterface())
318  return std::nullopt;
319 
320  return entry.getValue();
321 }
322 
325  TypeID typeID) {
326  return llvm::filter_to_vector<4>(
327  entries, [typeID](DataLayoutEntryInterface entry) {
328  auto type = llvm::dyn_cast_if_present<Type>(entry.getKey());
329  return type && type.getTypeID() == typeID;
330  });
331 }
332 
333 DataLayoutEntryInterface
335  StringAttr id) {
336  const auto *it = llvm::find_if(entries, [id](DataLayoutEntryInterface entry) {
337  if (auto attr = dyn_cast<StringAttr>(entry.getKey()))
338  return attr == id;
339  return false;
340  });
341  return it == entries.end() ? DataLayoutEntryInterface() : *it;
342 }
343 
344 static DataLayoutSpecInterface getSpec(Operation *operation) {
346  .Case<ModuleOp, DataLayoutOpInterface>(
347  [&](auto op) { return op.getDataLayoutSpec(); })
348  .Default([](Operation *) {
349  llvm_unreachable("expected an op with data layout spec");
350  return DataLayoutSpecInterface();
351  });
352 }
353 
354 static TargetSystemSpecInterface getTargetSystemSpec(Operation *operation) {
355  if (operation) {
356  ModuleOp moduleOp = dyn_cast<ModuleOp>(operation);
357  if (!moduleOp)
358  moduleOp = operation->getParentOfType<ModuleOp>();
359  return moduleOp.getTargetSystemSpec();
360  }
361  return TargetSystemSpecInterface();
362 }
363 
364 /// Populates `opsWithLayout` with the list of proper ancestors of `leaf` that
365 /// are either modules or implement the `DataLayoutOpInterface`.
366 static void
369  SmallVectorImpl<Location> *opLocations = nullptr) {
370  if (!leaf)
371  return;
372 
373  for (Operation *parent = leaf->getParentOp(); parent != nullptr;
374  parent = parent->getParentOp()) {
376  .Case<ModuleOp>([&](ModuleOp op) {
377  // Skip top-level module op unless it has a layout. Top-level module
378  // without layout is most likely the one implicitly added by the
379  // parser and it doesn't have location. Top-level null specification
380  // would have had the same effect as not having a specification at all
381  // (using type defaults).
382  if (!op->getParentOp() && !op.getDataLayoutSpec())
383  return;
384  specs.push_back(op.getDataLayoutSpec());
385  if (opLocations)
386  opLocations->push_back(op.getLoc());
387  })
388  .Case<DataLayoutOpInterface>([&](DataLayoutOpInterface op) {
389  specs.push_back(op.getDataLayoutSpec());
390  if (opLocations)
391  opLocations->push_back(op.getLoc());
392  });
393  }
394 }
395 
396 /// Returns a layout spec that is a combination of the layout specs attached
397 /// to the given operation and all its ancestors.
398 static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf) {
399  if (!leaf)
400  return {};
401 
402  assert((isa<ModuleOp, DataLayoutOpInterface>(leaf)) &&
403  "expected an op with data layout spec");
404 
407  collectParentLayouts(leaf, specs);
408 
409  // Fast track if there are no ancestors.
410  if (specs.empty())
411  return getSpec(leaf);
412 
413  // Create the list of non-null specs (null/missing specs can be safely
414  // ignored) from the outermost to the innermost.
415  auto nonNullSpecs = llvm::filter_to_vector<2>(
416  llvm::reverse(specs),
417  [](DataLayoutSpecInterface iface) { return iface != nullptr; });
418 
419  // Combine the specs using the innermost as anchor.
420  if (DataLayoutSpecInterface current = getSpec(leaf))
421  return current.combineWith(nonNullSpecs);
422  if (nonNullSpecs.empty())
423  return {};
424  return nonNullSpecs.back().combineWith(
425  llvm::ArrayRef(nonNullSpecs).drop_back());
426 }
427 
429  DataLayoutSpecInterface spec = getSpec(op);
430  // The layout specification may be missing and it's fine.
431  if (!spec)
432  return success();
433 
434  if (failed(spec.verifySpec(op->getLoc())))
435  return failure();
436  if (!getCombinedDataLayout(op)) {
438  op->emitError()
439  << "data layout does not combine with layouts of enclosing ops";
441  SmallVector<Location> opLocations;
442  collectParentLayouts(op, specs, &opLocations);
443  for (Location loc : opLocations)
444  diag.attachNote(loc) << "enclosing op with data layout";
445  return diag;
446  }
447  return success();
448 }
449 
450 llvm::TypeSize mlir::detail::divideCeil(llvm::TypeSize numerator,
451  uint64_t denominator) {
452  uint64_t divided =
453  llvm::divideCeil(numerator.getKnownMinValue(), denominator);
454  return llvm::TypeSize::get(divided, numerator.isScalable());
455 }
456 
457 //===----------------------------------------------------------------------===//
458 // DataLayout
459 //===----------------------------------------------------------------------===//
460 
461 template <typename OpTy>
462 void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op) {
463  if (!originalLayout) {
464  assert((!op || !op.getDataLayoutSpec()) &&
465  "could not compute layout information for an op (failed to "
466  "combine attributes?)");
467  }
468 }
469 
471 
472 mlir::DataLayout::DataLayout(DataLayoutOpInterface op)
473  : originalLayout(getCombinedDataLayout(op)),
474  originalTargetSystemDesc(getTargetSystemSpec(op)), scope(op),
475  allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
476  globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
477 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
478  checkMissingLayout(originalLayout, op);
479  collectParentLayouts(op, layoutStack);
480 #endif
481 }
482 
484  : originalLayout(getCombinedDataLayout(op)),
485  originalTargetSystemDesc(getTargetSystemSpec(op)), scope(op),
486  allocaMemorySpace(std::nullopt), programMemorySpace(std::nullopt),
487  globalMemorySpace(std::nullopt), stackAlignment(std::nullopt) {
488 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
489  checkMissingLayout(originalLayout, op);
490  collectParentLayouts(op, layoutStack);
491 #endif
492 }
493 
495  // Search the closest parent either being a module operation or implementing
496  // the data layout interface.
497  while (op) {
498  if (auto module = dyn_cast<ModuleOp>(op))
499  return DataLayout(module);
500  if (auto iface = dyn_cast<DataLayoutOpInterface>(op))
501  return DataLayout(iface);
502  op = op->getParentOp();
503  }
504  return DataLayout();
505 }
506 
507 void mlir::DataLayout::checkValid() const {
508 #if LLVM_ENABLE_ABI_BREAKING_CHECKS
510  collectParentLayouts(scope, specs);
511  assert(specs.size() == layoutStack.size() &&
512  "data layout object used, but no longer valid due to the change in "
513  "number of nested layouts");
514  for (auto pair : llvm::zip(specs, layoutStack)) {
515  Attribute newLayout = std::get<0>(pair);
516  Attribute origLayout = std::get<1>(pair);
517  assert(newLayout == origLayout &&
518  "data layout object used, but no longer valid "
519  "due to the change in layout attributes");
520  }
521 #endif
522  assert(((!scope && !this->originalLayout) ||
523  (scope && this->originalLayout == getCombinedDataLayout(scope))) &&
524  "data layout object used, but no longer valid due to the change in "
525  "layout spec");
526 }
527 
528 /// Looks up the value for the given type key in the given cache. If there is no
529 /// such value in the cache, compute it using the given callback and put it in
530 /// the cache before returning.
531 template <typename T>
533  function_ref<T(Type)> compute) {
534  auto it = cache.find(t);
535  if (it != cache.end())
536  return it->second;
537 
538  auto result = cache.try_emplace(t, compute(t));
539  return result.first->second;
540 }
541 
542 llvm::TypeSize mlir::DataLayout::getTypeSize(Type t) const {
543  checkValid();
544  return cachedLookup<llvm::TypeSize>(t, sizes, [&](Type ty) {
545  DataLayoutEntryList list;
546  if (originalLayout)
547  list = originalLayout.getSpecForType(ty.getTypeID());
548  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
549  return iface.getTypeSize(ty, *this, list);
550  return detail::getDefaultTypeSize(ty, *this, list);
551  });
552 }
553 
554 llvm::TypeSize mlir::DataLayout::getTypeSizeInBits(Type t) const {
555  checkValid();
556  return cachedLookup<llvm::TypeSize>(t, bitsizes, [&](Type ty) {
557  DataLayoutEntryList list;
558  if (originalLayout)
559  list = originalLayout.getSpecForType(ty.getTypeID());
560  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
561  return iface.getTypeSizeInBits(ty, *this, list);
562  return detail::getDefaultTypeSizeInBits(ty, *this, list);
563  });
564 }
565 
567  checkValid();
568  return cachedLookup<uint64_t>(t, abiAlignments, [&](Type ty) {
569  DataLayoutEntryList list;
570  if (originalLayout)
571  list = originalLayout.getSpecForType(ty.getTypeID());
572  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
573  return iface.getTypeABIAlignment(ty, *this, list);
574  return detail::getDefaultABIAlignment(ty, *this, list);
575  });
576 }
577 
579  checkValid();
580  return cachedLookup<uint64_t>(t, preferredAlignments, [&](Type ty) {
581  DataLayoutEntryList list;
582  if (originalLayout)
583  list = originalLayout.getSpecForType(ty.getTypeID());
584  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
585  return iface.getTypePreferredAlignment(ty, *this, list);
586  return detail::getDefaultPreferredAlignment(ty, *this, list);
587  });
588 }
589 
590 std::optional<uint64_t> mlir::DataLayout::getTypeIndexBitwidth(Type t) const {
591  checkValid();
592  return cachedLookup<std::optional<uint64_t>>(t, indexBitwidths, [&](Type ty) {
593  DataLayoutEntryList list;
594  if (originalLayout)
595  list = originalLayout.getSpecForType(ty.getTypeID());
596  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
597  return iface.getIndexBitwidth(ty, *this, list);
598  return detail::getDefaultIndexBitwidth(ty, *this, list);
599  });
600 }
601 
603  checkValid();
604  if (endianness)
605  return *endianness;
606  DataLayoutEntryInterface entry;
607  if (originalLayout)
608  entry = originalLayout.getSpecForIdentifier(
609  originalLayout.getEndiannessIdentifier(originalLayout.getContext()));
610 
611  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
612  endianness = iface.getEndianness(entry);
613  else
614  endianness = detail::getDefaultEndianness(entry);
615  return *endianness;
616 }
617 
619  checkValid();
620  if (defaultMemorySpace)
621  return *defaultMemorySpace;
622  DataLayoutEntryInterface entry;
623  if (originalLayout)
624  entry = originalLayout.getSpecForIdentifier(
625  originalLayout.getDefaultMemorySpaceIdentifier(
626  originalLayout.getContext()));
627  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
628  defaultMemorySpace = iface.getDefaultMemorySpace(entry);
629  else
630  defaultMemorySpace = detail::getDefaultMemorySpace(entry);
631  return *defaultMemorySpace;
632 }
633 
635  checkValid();
636  if (allocaMemorySpace)
637  return *allocaMemorySpace;
638  DataLayoutEntryInterface entry;
639  if (originalLayout)
640  entry = originalLayout.getSpecForIdentifier(
641  originalLayout.getAllocaMemorySpaceIdentifier(
642  originalLayout.getContext()));
643  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
644  allocaMemorySpace = iface.getAllocaMemorySpace(entry);
645  else
646  allocaMemorySpace = detail::getDefaultAllocaMemorySpace(entry);
647  return *allocaMemorySpace;
648 }
649 
651  checkValid();
652  if (manglingMode)
653  return *manglingMode;
654  DataLayoutEntryInterface entry;
655  if (originalLayout)
656  entry = originalLayout.getSpecForIdentifier(
657  originalLayout.getManglingModeIdentifier(originalLayout.getContext()));
658 
659  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
660  manglingMode = iface.getManglingMode(entry);
661  else
662  manglingMode = detail::getDefaultManglingMode(entry);
663  return *manglingMode;
664 }
665 
667  checkValid();
668  if (programMemorySpace)
669  return *programMemorySpace;
670  DataLayoutEntryInterface entry;
671  if (originalLayout)
672  entry = originalLayout.getSpecForIdentifier(
673  originalLayout.getProgramMemorySpaceIdentifier(
674  originalLayout.getContext()));
675  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
676  programMemorySpace = iface.getProgramMemorySpace(entry);
677  else
678  programMemorySpace = detail::getDefaultProgramMemorySpace(entry);
679  return *programMemorySpace;
680 }
681 
683  checkValid();
684  if (globalMemorySpace)
685  return *globalMemorySpace;
686  DataLayoutEntryInterface entry;
687  if (originalLayout)
688  entry = originalLayout.getSpecForIdentifier(
689  originalLayout.getGlobalMemorySpaceIdentifier(
690  originalLayout.getContext()));
691  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
692  globalMemorySpace = iface.getGlobalMemorySpace(entry);
693  else
694  globalMemorySpace = detail::getDefaultGlobalMemorySpace(entry);
695  return *globalMemorySpace;
696 }
697 
699  checkValid();
700  if (stackAlignment)
701  return *stackAlignment;
702  DataLayoutEntryInterface entry;
703  if (originalLayout)
704  entry = originalLayout.getSpecForIdentifier(
705  originalLayout.getStackAlignmentIdentifier(
706  originalLayout.getContext()));
707  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
708  stackAlignment = iface.getStackAlignment(entry);
709  else
710  stackAlignment = detail::getDefaultStackAlignment(entry);
711  return *stackAlignment;
712 }
713 
715  TargetSystemSpecInterface::DeviceID deviceID,
716  StringAttr propertyName) const {
717  checkValid();
718  DataLayoutEntryInterface entry;
719  if (originalTargetSystemDesc) {
720  if (std::optional<TargetDeviceSpecInterface> device =
721  originalTargetSystemDesc.getDeviceSpecForDeviceID(deviceID))
722  entry = device->getSpecForIdentifier(propertyName);
723  }
724  // Currently I am not caching the results because we do not return
725  // default values of these properties. Instead if the property is
726  // missing, we return std::nullopt so that the users can resort to
727  // the default value however they want.
728  if (auto iface = dyn_cast_or_null<DataLayoutOpInterface>(scope))
729  return iface.getDevicePropertyValue(entry);
730  else
731  return detail::getDevicePropertyValue(entry);
732 }
733 
734 //===----------------------------------------------------------------------===//
735 // DataLayoutSpecInterface
736 //===----------------------------------------------------------------------===//
737 
738 void DataLayoutSpecInterface::bucketEntriesByType(
741  for (DataLayoutEntryInterface entry : getEntries()) {
742  if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey()))
743  types[type.getTypeID()].push_back(entry);
744  else
745  ids[llvm::cast<StringAttr>(entry.getKey())] = entry;
746  }
747 }
748 
749 LogicalResult mlir::detail::verifyDataLayoutSpec(DataLayoutSpecInterface spec,
750  Location loc) {
751  // First, verify individual entries.
752  for (DataLayoutEntryInterface entry : spec.getEntries())
753  if (failed(entry.verifyEntry(loc)))
754  return failure();
755 
756  // Second, dispatch verifications of entry groups to types or dialects they
757  // are associated with.
760  spec.bucketEntriesByType(types, ids);
761 
762  for (const auto &kvp : types) {
763  auto sampleType = cast<Type>(kvp.second.front().getKey());
764  if (isa<IndexType>(sampleType)) {
765  assert(kvp.second.size() == 1 &&
766  "expected one data layout entry for non-parametric 'index' type");
767  if (!isa<IntegerAttr>(kvp.second.front().getValue()))
768  return emitError(loc)
769  << "expected integer attribute in the data layout entry for "
770  << sampleType;
771  continue;
772  }
773 
774  if (sampleType.isIntOrFloat()) {
775  for (DataLayoutEntryInterface entry : kvp.second) {
776  auto value = dyn_cast<DenseIntElementsAttr>(entry.getValue());
777  if (!value || !value.getElementType().isSignlessInteger(64)) {
778  emitError(loc) << "expected a dense i64 elements attribute in the "
779  "data layout entry "
780  << entry;
781  return failure();
782  }
783 
784  auto elements = llvm::to_vector<2>(value.getValues<uint64_t>());
785  unsigned numElements = elements.size();
786  if (numElements < 1 || numElements > 2) {
787  emitError(loc) << "expected 1 or 2 elements in the data layout entry "
788  << entry;
789  return failure();
790  }
791 
792  uint64_t abi = elements[0];
793  uint64_t preferred = numElements == 2 ? elements[1] : abi;
794  if (preferred < abi) {
795  emitError(loc)
796  << "preferred alignment is expected to be greater than or equal "
797  "to the abi alignment in data layout entry "
798  << entry;
799  return failure();
800  }
801  }
802  continue;
803  }
804 
805  if (isa<BuiltinDialect>(&sampleType.getDialect()))
806  return emitError(loc) << "unexpected data layout for a built-in type";
807 
808  auto dlType = dyn_cast<DataLayoutTypeInterface>(sampleType);
809  if (!dlType)
810  return emitError(loc)
811  << "data layout specified for a type that does not support it";
812  if (failed(dlType.verifyEntries(kvp.second, loc)))
813  return failure();
814  }
815 
816  for (const auto &kvp : ids) {
817  StringAttr identifier = cast<StringAttr>(kvp.second.getKey());
818  Dialect *dialect = identifier.getReferencedDialect();
819 
820  // Ignore attributes that belong to an unknown dialect, the dialect may
821  // actually implement the relevant interface but we don't know about that.
822  if (!dialect)
823  continue;
824 
825  const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
826  if (!iface) {
827  return emitError(loc)
828  << "the '" << dialect->getNamespace()
829  << "' dialect does not support identifier data layout entries";
830  }
831  if (failed(iface->verifyEntry(kvp.second, loc)))
832  return failure();
833  }
834 
835  return success();
836 }
837 
838 LogicalResult
839 mlir::detail::verifyTargetSystemSpec(TargetSystemSpecInterface spec,
840  Location loc) {
843  for (const auto &entry : spec.getEntries()) {
844  auto targetDeviceSpec =
845  dyn_cast<TargetDeviceSpecInterface>(entry.getValue());
846 
847  if (!targetDeviceSpec)
848  return failure();
849 
850  // First, verify individual target device desc specs.
851  if (failed(targetDeviceSpec.verifyEntry(loc)))
852  return failure();
853 
854  // Check that device IDs are unique across all entries.
855  auto deviceID =
856  llvm::dyn_cast<TargetSystemSpecInterface::DeviceID>(entry.getKey());
857  if (!deviceID)
858  return failure();
859 
860  if (!deviceIDs.insert(deviceID).second) {
861  return failure();
862  }
863 
864  // collect all the keys used by all the target device specs.
865  for (DataLayoutEntryInterface entry : targetDeviceSpec.getEntries()) {
866  if (auto type = llvm::dyn_cast_if_present<Type>(entry.getKey())) {
867  // targetDeviceSpec does not support Type as a key.
868  return failure();
869  } else {
870  deviceDescKeys[cast<StringAttr>(entry.getKey())] = entry;
871  }
872  }
873  }
874 
875  for (const auto &[keyName, keyVal] : deviceDescKeys) {
876  Dialect *dialect = keyName.getReferencedDialect();
877 
878  // Ignore attributes that belong to an unknown dialect, the dialect may
879  // actually implement the relevant interface but we don't know about that.
880  if (!dialect)
881  return failure();
882 
883  const auto *iface = dyn_cast<DataLayoutDialectInterface>(dialect);
884  if (!iface) {
885  return emitError(loc)
886  << "the '" << dialect->getNamespace()
887  << "' dialect does not support identifier data layout entries";
888  }
889  if (failed(iface->verifyEntry(keyVal, loc)))
890  return failure();
891  }
892 
893  return success();
894 }
895 
896 #include "mlir/Interfaces/DataLayoutAttrInterface.cpp.inc"
897 #include "mlir/Interfaces/DataLayoutOpInterface.cpp.inc"
898 #include "mlir/Interfaces/DataLayoutTypeInterface.cpp.inc"
static uint64_t getIntegerTypePreferredAlignment(IntegerType intType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getCombinedDataLayout(Operation *leaf)
Returns a layout spec that is a combination of the layout specs attached to the given operation and a...
static TargetSystemSpecInterface getTargetSystemSpec(Operation *operation)
constexpr static const uint64_t kDefaultBitsInByte
static uint64_t getIntegerTypeABIAlignment(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
static T cachedLookup(Type t, DenseMap< Type, T > &cache, function_ref< T(Type)> compute)
Looks up the value for the given type key in the given cache.
static uint64_t extractPreferredAlignment(DataLayoutEntryInterface entry)
static void collectParentLayouts(Operation *leaf, SmallVectorImpl< DataLayoutSpecInterface > &specs, SmallVectorImpl< Location > *opLocations=nullptr)
Populates opsWithLayout with the list of proper ancestors of leaf that are either modules or implemen...
static uint64_t getFloatTypePreferredAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static uint64_t getFloatTypeABIAlignment(FloatType fltType, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
static DataLayoutSpecInterface getSpec(Operation *operation)
static uint64_t getIndexBitwidth(DataLayoutEntryListRef params)
Returns the bitwidth of the index type if specified in the param list.
static uint64_t extractABIAlignment(DataLayoutEntryInterface entry)
static DataLayoutEntryInterface findEntryForIntegerType(IntegerType intType, ArrayRef< DataLayoutEntryInterface > params)
void checkMissingLayout(DataLayoutSpecInterface originalLayout, OpTy op)
static void reportMissingDataLayout(Type type)
Reports that the given type is missing the data layout information and exits.
static std::string diag(const llvm::Value &value)
Attributes are known-constant values of operations.
Definition: Attributes.h:25
The main mechanism for performing data layout queries.
Attribute getAllocaMemorySpace() const
Returns the memory space used for AllocaOps.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
std::optional< uint64_t > getTypeIndexBitwidth(Type t) const
Returns the bitwidth that should be used when performing index computations for the given pointer-lik...
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
Attribute getManglingMode() const
Returns the mangling mode.
uint64_t getStackAlignment() const
Returns the natural alignment of the stack in bits.
Attribute getProgramMemorySpace() const
Returns the memory space used for program memory operations.
uint64_t getTypePreferredAlignment(Type t) const
Returns the preferred of the given type in the current scope.
Attribute getGlobalMemorySpace() const
Returns the memory space used for global operations.
uint64_t getTypeABIAlignment(Type t) const
Returns the required alignment of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Attribute getDefaultMemorySpace() const
Returns the default memory space used for memory operations.
Attribute getEndianness() const
Returns the specified endianness.
std::optional< Attribute > getDevicePropertyValue(TargetSystemSpecInterface::DeviceID, StringAttr propertyName) const
Returns the value of the specified property if the property is defined for the given device ID,...
Dialects are groups of MLIR operations, types and attributes, as well as behavior associated with the...
Definition: Dialect.h:38
StringRef getNamespace() const
Definition: Dialect.h:54
This class represents a diagnostic that is inflight and set to be reported.
Definition: Diagnostics.h:314
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition: Location.h:66
Operation is the basic unit of execution within MLIR.
Definition: Operation.h:88
Location getLoc()
The source location the operation was defined or derived from.
Definition: Operation.h:223
Operation * getParentOp()
Returns the closest surrounding operation that contains this operation or nullptr if this is a top-le...
Definition: Operation.h:234
InFlightDiagnostic emitError(const Twine &message={})
Emit an error about fatal conditions with this operation, reporting up to any diagnostic handlers tha...
Definition: Operation.cpp:268
OpTy getParentOfType()
Return the closest surrounding parent operation that is of type 'OpTy'.
Definition: Operation.h:238
This class provides an efficient unique identifier for a specific C++ type.
Definition: TypeID.h:107
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition: Types.h:74
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
Definition: Types.cpp:35
bool isIntOrFloat() const
Return true if this is an integer (of any signedness) or a float type.
Definition: Types.cpp:116
TypeID getTypeID()
Return a unique identifier for the concrete type.
Definition: Types.h:117
unsigned getIntOrFloatBitWidth() const
Return the bit width of an integer or a float type, assert failure on other types.
Definition: Types.cpp:122
Attribute getDefaultAllocaMemorySpace(DataLayoutEntryInterface entry)
Default handler for alloca memory space request.
Attribute getDefaultProgramMemorySpace(DataLayoutEntryInterface entry)
Default handler for program memory space request.
Attribute getDefaultEndianness(DataLayoutEntryInterface entry)
Default handler for endianness request.
std::optional< uint64_t > getDefaultIndexBitwidth(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the index bitwidth request.
DataLayoutEntryList filterEntriesForType(DataLayoutEntryListRef entries, TypeID typeID)
Given a list of data layout entries, returns a new list containing the entries with keys having the g...
LogicalResult verifyTargetSystemSpec(TargetSystemSpecInterface spec, Location loc)
Verifies that a target system desc spec is valid.
std::optional< Attribute > getDevicePropertyValue(DataLayoutEntryInterface entry)
Returns the value of the property from the specified DataLayoutEntry.
Attribute getDefaultManglingMode(DataLayoutEntryInterface entry)
Default handler for mangling mode request.
uint64_t getDefaultABIAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the required alignment request.
llvm::TypeSize getDefaultTypeSize(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size request.
llvm::TypeSize getDefaultTypeSizeInBits(Type type, const DataLayout &dataLayout, DataLayoutEntryListRef params)
Default handler for the type size in bits request.
uint64_t getDefaultPreferredAlignment(Type type, const DataLayout &dataLayout, ArrayRef< DataLayoutEntryInterface > params)
Default handler for the preferred alignment request.
llvm::TypeSize divideCeil(llvm::TypeSize numerator, uint64_t denominator)
Divides the known min value of the numerator by the denominator and rounds the result up to the next ...
uint64_t getDefaultStackAlignment(DataLayoutEntryInterface entry)
Default handler for the stack alignment request.
Attribute getDefaultGlobalMemorySpace(DataLayoutEntryInterface entry)
Default handler for global memory space request.
Attribute getDefaultMemorySpace(DataLayoutEntryInterface entry)
Default handler for the default memory space request.
LogicalResult verifyDataLayoutOp(Operation *op)
Verifies that the operation implementing the data layout interface, or a module operation,...
LogicalResult verifyDataLayoutSpec(DataLayoutSpecInterface spec, Location loc)
Verifies that a data layout spec is valid.
DataLayoutEntryInterface filterEntryForIdentifier(DataLayoutEntryListRef entries, StringAttr id)
Given a list of data layout entries, returns the entry that has the given identifier as key,...
Include the generated interface declarations.
InFlightDiagnostic emitError(Location loc)
Utility method to emit an error message using this location.
auto get(MLIRContext *context, Ts &&...params)
Helper method that injects context only if needed, this helps unify some of the attribute constructio...