MLIR 24.0.0git
LLVMMemoryOps.cpp
Go to the documentation of this file.
1//===- LLVMMemoryOps.cpp - LLVM dialect memory ops ------------------------===//
2//
3// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4// See https://llvm.org/LICENSE.txt for license information.
5// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6//
7//===----------------------------------------------------------------------===//
8//
9// This file defines the ops of op partition `Memory` of the LLVM dialect.
10//
11//===----------------------------------------------------------------------===//
12
14
15#include "IR/LLVMOps.h"
20#include "mlir/IR/Builders.h"
22#include "mlir/IR/Matchers.h"
26#include "llvm/ADT/STLExtras.h"
27#include "llvm/ADT/TypeSwitch.h"
28#include "llvm/Support/DebugLog.h"
29
30#include <optional>
31
32#define DEBUG_TYPE "sroa"
33
34using namespace mlir;
35using namespace mlir::LLVM;
36
37//===----------------------------------------------------------------------===//
38// Attribute Helpers
39//===----------------------------------------------------------------------===//
40
41static constexpr const char kElemTypeAttrName[] = "elem_type";
42
43//===----------------------------------------------------------------------===//
44// Printing, parsing, verification and canonicalization for LLVM::AllocaOp.
45//===----------------------------------------------------------------------===//
46
47void AllocaOp::print(OpAsmPrinter &p) {
48 auto funcTy =
49 FunctionType::get(getContext(), {getArraySize().getType()}, {getType()});
50
51 if (getInalloca())
52 p << " inalloca";
53
54 p << ' ' << getArraySize() << " x " << getElemType();
55 NamedAttrList attrs((*this)->getDiscardableAttrDictionary().getValue());
56 if (getAlignment() && *getAlignment() != 0)
57 attrs.append(getAlignmentAttrName(), getAlignmentAttr());
58 p.printOptionalAttrDict(attrs);
59 p << " : " << funcTy;
60}
61
62// <operation> ::= `llvm.alloca` `inalloca`? ssa-use `x` type
63// attribute-dict? `:` type `,` type
64ParseResult AllocaOp::parse(OpAsmParser &parser, OperationState &result) {
66 Type type, elemType;
67 SMLoc trailingTypeLoc;
68
69 if (succeeded(parser.parseOptionalKeyword("inalloca")))
70 result.addAttribute(getInallocaAttrName(result.name),
71 UnitAttr::get(parser.getContext()));
72
73 if (parser.parseOperand(arraySize) || parser.parseKeyword("x") ||
74 parser.parseType(elemType) ||
75 parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
76 parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type))
77 return failure();
78
79 std::optional<NamedAttribute> alignmentAttr =
80 result.attributes.getNamed("alignment");
81 if (alignmentAttr.has_value()) {
82 auto alignmentInt = llvm::dyn_cast<IntegerAttr>(alignmentAttr->getValue());
83 if (!alignmentInt)
84 return parser.emitError(parser.getNameLoc(),
85 "expected integer alignment");
86 if (alignmentInt.getValue().isZero())
87 result.attributes.erase("alignment");
88 }
89
90 // Extract the result type from the trailing function type.
91 auto funcType = llvm::dyn_cast<FunctionType>(type);
92 if (!funcType || funcType.getNumInputs() != 1 ||
93 funcType.getNumResults() != 1)
94 return parser.emitError(
95 trailingTypeLoc,
96 "expected trailing function type with one argument and one result");
97
98 if (parser.resolveOperand(arraySize, funcType.getInput(0), result.operands))
99 return failure();
100
101 Type resultType = funcType.getResult(0);
102 if (auto ptrResultType = llvm::dyn_cast<LLVMPointerType>(resultType))
103 result.addAttribute(kElemTypeAttrName, TypeAttr::get(elemType));
104
105 result.addTypes({funcType.getResult(0)});
106 return success();
107}
108
109LogicalResult AllocaOp::verify() {
110 // Only certain target extension types can be used in 'alloca'.
111 if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getElemType());
112 targetExtType && !targetExtType.supportsMemOps())
113 return emitOpError()
114 << "this target extension type cannot be used in alloca";
115
116 return success();
117}
118
119LogicalResult AllocaOp::canonicalize(AllocaOp op, PatternRewriter &rewriter) {
120 // Convert `alloca Ty, C` to the canonical `alloca [C x Ty], 1` form.
121 APInt numElements;
122 if (!matchPattern(op.getArraySize(), m_ConstantInt(&numElements)) ||
123 numElements.isOne() || numElements.getActiveBits() > 64)
124 return failure();
125
126 auto arrayType =
127 LLVMArrayType::get(op.getElemType(), numElements.getZExtValue());
128 Value one = ConstantOp::create(rewriter, op.getLoc(), rewriter.getI32Type(),
129 /*value=*/1);
130 auto newAlloca =
131 AllocaOp::create(rewriter, op.getLoc(), op.getType(), one,
132 op.getAlignmentAttr(), arrayType, op.getInalloca());
133 newAlloca->setDiscardableAttrs(op->getDiscardableAttrDictionary());
134 rewriter.replaceOp(op, newAlloca);
135 return success();
136}
137
138//===----------------------------------------------------------------------===//
139// Code for LLVM::GEPOp.
140//===----------------------------------------------------------------------===//
141
142GEPIndicesAdaptor<ValueRange> GEPOp::getIndices() {
143 return GEPIndicesAdaptor<ValueRange>(getRawConstantIndicesAttr(),
144 getDynamicIndices());
145}
146
147/// Returns the elemental type of any LLVM-compatible vector type or self.
149 if (auto vectorType = llvm::dyn_cast<VectorType>(type))
150 return vectorType.getElementType();
151 return type;
152}
153
154/// Destructures the 'indices' parameter into 'rawConstantIndices' and
155/// 'dynamicIndices', encoding the former in the process. In the process,
156/// dynamic indices which are used to index into a structure type are converted
157/// to constant indices when possible. To do this, the GEPs element type should
158/// be passed as first parameter.
160 SmallVectorImpl<int32_t> &rawConstantIndices,
161 SmallVectorImpl<Value> &dynamicIndices) {
162 for (const GEPArg &iter : indices) {
163 // If the thing we are currently indexing into is a struct we must turn
164 // any integer constants into constant indices. If this is not possible
165 // we don't do anything here. The verifier will catch it and emit a proper
166 // error. All other canonicalization is done in the fold method.
167 bool requiresConst = !rawConstantIndices.empty() &&
168 isa_and_nonnull<LLVMStructType>(currType);
169 if (Value val = llvm::dyn_cast_if_present<Value>(iter)) {
170 APInt intC;
171 if (requiresConst && matchPattern(val, m_ConstantInt(&intC)) &&
172 intC.isSignedIntN(kGEPConstantBitWidth)) {
173 rawConstantIndices.push_back(intC.getSExtValue());
174 } else {
175 rawConstantIndices.push_back(GEPOp::kDynamicIndex);
176 dynamicIndices.push_back(val);
177 }
178 } else {
179 rawConstantIndices.push_back(cast<GEPConstantIndex>(iter));
180 }
181
182 // Skip for very first iteration of this loop. First index does not index
183 // within the aggregates, but is just a pointer offset.
184 if (rawConstantIndices.size() == 1 || !currType)
185 continue;
186
187 currType = TypeSwitch<Type, Type>(currType)
188 .Case<VectorType, LLVMArrayType>([](auto containerType) {
189 return containerType.getElementType();
190 })
191 .Case([&](LLVMStructType structType) -> Type {
192 int64_t memberIndex = rawConstantIndices.back();
193 if (memberIndex >= 0 && static_cast<size_t>(memberIndex) <
194 structType.getBody().size())
195 return structType.getBody()[memberIndex];
196 return nullptr;
197 })
198 .Default(nullptr);
199 }
200}
201
202void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
203 Type elementType, Value basePtr, ArrayRef<GEPArg> indices,
204 GEPNoWrapFlags noWrapFlags, ConstantRangeAttr inrange,
205 ArrayRef<NamedAttribute> attributes) {
206 SmallVector<int32_t> rawConstantIndices;
207 SmallVector<Value> dynamicIndices;
208 destructureIndices(elementType, indices, rawConstantIndices, dynamicIndices);
209
210 result.addTypes(resultType);
211 result.addAttributes(attributes);
212 result.getOrAddProperties<Properties>().rawConstantIndices =
213 builder.getDenseI32ArrayAttr(rawConstantIndices);
214 result.getOrAddProperties<Properties>().noWrapFlags = noWrapFlags;
215 result.getOrAddProperties<Properties>().elem_type =
216 TypeAttr::get(elementType);
217 result.getOrAddProperties<Properties>().inrange = inrange;
218 result.addOperands(basePtr);
219 result.addOperands(dynamicIndices);
220}
221
222void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
223 Type elementType, Value basePtr, ValueRange indices,
224 GEPNoWrapFlags noWrapFlags, ConstantRangeAttr inrange,
225 ArrayRef<NamedAttribute> attributes) {
226 build(builder, result, resultType, elementType, basePtr,
227 SmallVector<GEPArg>(indices), noWrapFlags, inrange, attributes);
228}
229
231 OpAsmParser &parser,
233 DenseI32ArrayAttr &rawConstantIndices) {
234 SmallVector<int32_t> constantIndices;
235
236 auto idxParser = [&]() -> ParseResult {
237 int32_t constantIndex;
238 OptionalParseResult parsedInteger =
239 parser.parseOptionalInteger(constantIndex);
240 if (parsedInteger.has_value()) {
241 if (failed(parsedInteger.value()))
242 return failure();
243 constantIndices.push_back(constantIndex);
244 return success();
245 }
246
247 constantIndices.push_back(LLVM::GEPOp::kDynamicIndex);
248 return parser.parseOperand(indices.emplace_back());
249 };
250 if (parser.parseCommaSeparatedList(idxParser))
251 return failure();
252
253 rawConstantIndices =
254 DenseI32ArrayAttr::get(parser.getContext(), constantIndices);
255 return success();
256}
257
258void mlir::LLVM::printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp,
260 DenseI32ArrayAttr rawConstantIndices) {
261 llvm::interleaveComma(
262 GEPIndicesAdaptor<OperandRange>(rawConstantIndices, indices), printer,
264 if (Value val = llvm::dyn_cast_if_present<Value>(cst))
265 printer.printOperand(val);
266 else
267 printer << cast<IntegerAttr>(cst).getInt();
268 });
269}
270
271/// For the given `indices`, check if they comply with `baseGEPType`,
272/// especially check against LLVMStructTypes nested within.
273static LogicalResult
274verifyStructIndices(Type baseGEPType, unsigned indexPos,
276 function_ref<InFlightDiagnostic()> emitOpError) {
277 if (indexPos >= indices.size())
278 // Stop searching
279 return success();
280
281 return TypeSwitch<Type, LogicalResult>(baseGEPType)
282 .Case([&](LLVMStructType structType) -> LogicalResult {
283 auto attr = dyn_cast<IntegerAttr>(indices[indexPos]);
284 if (!attr)
285 return emitOpError() << "expected index " << indexPos
286 << " indexing a struct to be constant";
287
288 int32_t gepIndex = attr.getInt();
289 ArrayRef<Type> elementTypes = structType.getBody();
290 if (gepIndex < 0 ||
291 static_cast<size_t>(gepIndex) >= elementTypes.size())
292 return emitOpError() << "index " << indexPos
293 << " indexing a struct is out of bounds";
294
295 // Instead of recursively going into every children types, we only
296 // dive into the one indexed by gepIndex.
297 return verifyStructIndices(elementTypes[gepIndex], indexPos + 1,
298 indices, emitOpError);
299 })
300 .Case<VectorType, LLVMArrayType>(
301 [&](auto containerType) -> LogicalResult {
302 return verifyStructIndices(containerType.getElementType(),
303 indexPos + 1, indices, emitOpError);
304 })
305 .Default([&](auto otherType) -> LogicalResult {
306 return emitOpError()
307 << "type " << otherType << " cannot be indexed (index #"
308 << indexPos << ")";
309 });
310}
311
312/// Driver function around `verifyStructIndices`.
313static LogicalResult
315 function_ref<InFlightDiagnostic()> emitOpError) {
316 return verifyStructIndices(baseGEPType, /*indexPos=*/1, indices, emitOpError);
317}
318
319LogicalResult LLVM::GEPOp::verify() {
320 if (static_cast<size_t>(
321 llvm::count(getRawConstantIndices(), kDynamicIndex)) !=
322 getDynamicIndices().size())
323 return emitOpError("expected as many dynamic indices as specified in '")
324 << getRawConstantIndicesAttrName().getValue() << "'";
325
326 if (getNoWrapFlags() == GEPNoWrapFlags::inboundsFlag)
327 return emitOpError("'inbounds_flag' cannot be used directly.");
328
329 // LLVM stores `inrange` only on GetElementPtrConstantExpr, whose operands
330 // are already Constants. This dialect has one GEP operation, so proving the
331 // base and indices are constant expressions means walking SSA. A local check
332 // rejects valid cases such as a nested constant GEP base or a `ptrtoint`
333 // index.
334 if (auto inrange = getInrangeAttr()) {
335 auto pointerType =
336 cast<LLVMPointerType>(extractVectorElementType(getBase().getType()));
337 DataLayout dataLayout = DataLayout::closest(*this);
338 std::optional<uint64_t> indexWidth =
339 dataLayout.getTypeIndexBitwidth(pointerType);
340 assert(indexWidth && "pointers always return an index bitwidth");
341 if (inrange.getLower().getBitWidth() != *indexWidth)
342 return emitOpError("'inrange' bitwidth ")
343 << inrange.getLower().getBitWidth()
344 << " must match the pointer index bitwidth (" << *indexWidth
345 << ") specified in the datalayout";
346 if (inrange.getLower().sge(inrange.getUpper()))
347 return emitOpError("expected 'inrange' end to be larger than start");
348 }
349
350 return verifyStructIndices(getElemType(), getIndices(),
351 [&] { return emitOpError(); });
352}
353
354//===----------------------------------------------------------------------===//
355// LoadOp
356//===----------------------------------------------------------------------===//
357
358void LoadOp::getEffects(
360 &effects) {
361 effects.emplace_back(MemoryEffects::Read::get(), &getAddrMutable());
362 // Volatile operations can have target-specific read-write effects on
363 // memory besides the one referred to by the pointer operand.
364 // Similarly, atomic operations that are monotonic or stricter cause
365 // synchronization that from a language point-of-view, are arbitrary
366 // read-writes into memory.
367 if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
368 getOrdering() != AtomicOrdering::unordered)) {
369 effects.emplace_back(MemoryEffects::Write::get());
370 effects.emplace_back(MemoryEffects::Read::get());
371 }
372}
373
374/// Returns true if the given type is supported by atomic operations. All
375/// integer, float, and pointer types with a power-of-two bitsize and a minimal
376/// size of 8 bits are supported.
378 const DataLayout &dataLayout) {
379 if (!isa<IntegerType, LLVMPointerType>(type))
381 return false;
382
383 llvm::TypeSize bitWidth = dataLayout.getTypeSizeInBits(type);
384 if (bitWidth.isScalable())
385 return false;
386 // Needs to be at least 8 bits and a power of two.
387 return bitWidth >= 8 && (bitWidth & (bitWidth - 1)) == 0;
388}
389
390/// Verifies the attributes and the type of atomic memory access operations.
391template <typename OpTy>
392static LogicalResult
393verifyAtomicMemOp(OpTy memOp, Type valueType,
394 ArrayRef<AtomicOrdering> unsupportedOrderings) {
395 if (memOp.getOrdering() != AtomicOrdering::not_atomic) {
396 DataLayout dataLayout = DataLayout::closest(memOp);
397 if (!isTypeCompatibleWithAtomicOp(valueType, dataLayout))
398 return memOp.emitOpError("unsupported type ")
399 << valueType << " for atomic access";
400 if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering()))
401 return memOp.emitOpError("unsupported ordering '")
402 << stringifyAtomicOrdering(memOp.getOrdering()) << "'";
403 if (!memOp.getAlignment())
404 return memOp.emitOpError("expected alignment for atomic access");
405 return success();
406 }
407 if (memOp.getSyncscope())
408 return memOp.emitOpError(
409 "expected syncscope to be null for non-atomic access");
410 return success();
411}
412
413LogicalResult LoadOp::verify() {
414 Type valueType = getResult().getType();
415 return verifyAtomicMemOp(*this, valueType,
416 {AtomicOrdering::release, AtomicOrdering::acq_rel});
417}
418
419void LoadOp::build(OpBuilder &builder, OperationState &state, Type type,
420 Value addr, unsigned alignment, bool isVolatile,
421 bool isNonTemporal, bool isInvariant, bool isInvariantGroup,
422 bool isNoundef, AtomicOrdering ordering,
423 StringRef syncscope) {
424 build(builder, state, type, addr,
425 alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
426 isNonTemporal, isInvariant, isInvariantGroup, ordering,
427 syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
428 /*dereferenceable=*/nullptr, isNoundef, /*range=*/nullptr,
429 /*access_groups=*/nullptr,
430 /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr,
431 /*tbaa=*/nullptr);
432}
433
434//===----------------------------------------------------------------------===//
435// StoreOp
436//===----------------------------------------------------------------------===//
437
438void StoreOp::getEffects(
440 &effects) {
441 effects.emplace_back(MemoryEffects::Write::get(), &getAddrMutable());
442 // Volatile operations can have target-specific read-write effects on
443 // memory besides the one referred to by the pointer operand.
444 // Similarly, atomic operations that are monotonic or stricter cause
445 // synchronization that from a language point-of-view, are arbitrary
446 // read-writes into memory.
447 if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
448 getOrdering() != AtomicOrdering::unordered)) {
449 effects.emplace_back(MemoryEffects::Write::get());
450 effects.emplace_back(MemoryEffects::Read::get());
451 }
452}
453
454LogicalResult StoreOp::verify() {
455 Type valueType = getValue().getType();
456 return verifyAtomicMemOp(*this, valueType,
457 {AtomicOrdering::acquire, AtomicOrdering::acq_rel});
458}
459
460void StoreOp::build(OpBuilder &builder, OperationState &state, Value value,
461 Value addr, unsigned alignment, bool isVolatile,
462 bool isNonTemporal, bool isInvariantGroup,
463 AtomicOrdering ordering, StringRef syncscope) {
464 build(builder, state, value, addr,
465 alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
466 isNonTemporal, isInvariantGroup, ordering,
467 syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
468 /*access_groups=*/nullptr,
469 /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
470}
471
472//===----------------------------------------------------------------------===//
473// AtomicRMWOp
474//===----------------------------------------------------------------------===//
475
476void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
477 AtomicBinOp binOp, Value ptr, Value val,
478 AtomicOrdering ordering, StringRef syncscope,
479 unsigned alignment, bool isVolatile,
480 bool ignoreDenormalMode) {
481 build(builder, state, val.getType(), binOp, ptr, val, ordering,
482 !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
483 alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
484 ignoreDenormalMode,
485 /*access_groups=*/nullptr,
486 /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
487}
488
489LogicalResult AtomicRMWOp::verify() {
490 auto valType = getVal().getType();
491 if (isFloatingPointOperation()) {
492 if (isCompatibleVectorType(valType)) {
493 if (isScalableVectorType(valType))
494 return emitOpError("expected LLVM IR fixed vector type");
495 Type elemType = llvm::cast<VectorType>(valType).getElementType();
496 if (!isCompatibleFloatingPointType(elemType))
497 return emitOpError(
498 "expected LLVM IR floating point type for vector element");
499 } else if (!isCompatibleFloatingPointType(valType)) {
500 return emitOpError("expected LLVM IR floating point type");
501 }
502 } else if (getBinOp() == AtomicBinOp::xchg) {
503 DataLayout dataLayout = DataLayout::closest(*this);
504 if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
505 return emitOpError("unexpected LLVM IR type for 'xchg' bin_op");
506 } else {
507 auto intType = llvm::dyn_cast<IntegerType>(valType);
508 unsigned intBitWidth = intType ? intType.getWidth() : 0;
509 if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 &&
510 intBitWidth != 64)
511 return emitOpError("expected LLVM IR integer type");
512 }
513
514 if (getIgnoreDenormalMode() && !isFloatingPointOperation())
515 return emitOpError(
516 "expected floating-point operation with 'ignore_denormal_mode'");
517
518 if (static_cast<unsigned>(getOrdering()) <
519 static_cast<unsigned>(AtomicOrdering::monotonic))
520 return emitOpError() << "expected at least '"
521 << stringifyAtomicOrdering(AtomicOrdering::monotonic)
522 << "' ordering";
523
524 return success();
525}
526
527//===----------------------------------------------------------------------===//
528// AtomicCmpXchgOp
529//===----------------------------------------------------------------------===//
530
531/// Returns an LLVM struct type that contains a value type and a boolean type.
533 auto boolType = IntegerType::get(valType.getContext(), 1);
534 return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType});
535}
536
537void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state,
538 Value ptr, Value cmp, Value val,
539 AtomicOrdering successOrdering,
540 AtomicOrdering failureOrdering, StringRef syncscope,
541 unsigned alignment, bool isWeak, bool isVolatile) {
542 build(builder, state, getValAndBoolStructType(val.getType()), ptr, cmp, val,
543 successOrdering, failureOrdering,
544 !syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
545 alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak,
546 isVolatile, /*access_groups=*/nullptr,
547 /*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
548}
549
550LogicalResult AtomicCmpXchgOp::verify() {
551 auto ptrType = llvm::cast<LLVM::LLVMPointerType>(getPtr().getType());
552 if (!ptrType)
553 return emitOpError("expected LLVM IR pointer type for operand #0");
554 auto valType = getVal().getType();
555 DataLayout dataLayout = DataLayout::closest(*this);
556 if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
557 return emitOpError("unexpected LLVM IR type");
558 if (getSuccessOrdering() < AtomicOrdering::monotonic ||
559 getFailureOrdering() < AtomicOrdering::monotonic)
560 return emitOpError("ordering must be at least 'monotonic'");
561 if (getFailureOrdering() == AtomicOrdering::release ||
562 getFailureOrdering() == AtomicOrdering::acq_rel)
563 return emitOpError("failure ordering cannot be 'release' or 'acq_rel'");
564 return success();
565}
566
567//===----------------------------------------------------------------------===//
568// FenceOp
569//===----------------------------------------------------------------------===//
570
571void FenceOp::build(OpBuilder &builder, OperationState &state,
572 AtomicOrdering ordering, StringRef syncscope) {
573 build(builder, state, ordering,
574 syncscope.empty() ? nullptr : builder.getStringAttr(syncscope));
575}
576
577LogicalResult FenceOp::verify() {
578 if (getOrdering() == AtomicOrdering::not_atomic ||
579 getOrdering() == AtomicOrdering::unordered ||
580 getOrdering() == AtomicOrdering::monotonic)
581 return emitOpError("can be given only acquire, release, acq_rel, "
582 "and seq_cst orderings");
583 return success();
584}
585
586//===----------------------------------------------------------------------===//
587// Folder for LLVM::GEPOp
588//===----------------------------------------------------------------------===//
589
590OpFoldResult LLVM::GEPOp::fold(FoldAdaptor adaptor) {
591 GEPIndicesAdaptor<ArrayRef<Attribute>> indices(getRawConstantIndicesAttr(),
592 adaptor.getDynamicIndices());
593
594 // Avoid losing inrange information.
595 if (getInrangeAttr())
596 return {};
597
598 // gep %x:T, 0 -> %x
599 if (getBase().getType() == getType() && indices.size() == 1)
600 if (auto integer = llvm::dyn_cast_or_null<IntegerAttr>(indices[0]))
601 if (integer.getValue().isZero())
602 return getBase();
603
604 // Canonicalize any dynamic indices of constant value to constant indices.
605 bool changed = false;
606 SmallVector<GEPArg> gepArgs;
607 for (auto iter : llvm::enumerate(indices)) {
608 auto integer = llvm::dyn_cast_or_null<IntegerAttr>(iter.value());
609 // Constant indices can only be int32_t, so if integer does not fit we
610 // are forced to keep it dynamic, despite being a constant.
611 if (!indices.isDynamicIndex(iter.index()) || !integer ||
612 !integer.getValue().isSignedIntN(kGEPConstantBitWidth)) {
613
614 PointerUnion<IntegerAttr, Value> existing = getIndices()[iter.index()];
615 if (Value val = llvm::dyn_cast_if_present<Value>(existing))
616 gepArgs.emplace_back(val);
617 else
618 gepArgs.emplace_back(cast<IntegerAttr>(existing).getInt());
619
620 continue;
621 }
622
623 changed = true;
624 gepArgs.emplace_back(integer.getInt());
625 }
626 if (changed) {
627 SmallVector<int32_t> rawConstantIndices;
628 SmallVector<Value> dynamicIndices;
629 destructureIndices(getElemType(), gepArgs, rawConstantIndices,
630 dynamicIndices);
631
632 getDynamicIndicesMutable().assign(dynamicIndices);
633 setRawConstantIndices(rawConstantIndices);
634 return Value{*this};
635 }
636
637 return {};
638}
639
640Value LLVM::GEPOp::getViewSource() { return getBase(); }
641
642//===----------------------------------------------------------------------===//
643// Interfaces for AllocaOp
644//===----------------------------------------------------------------------===//
645
646llvm::SmallVector<MemorySlot> LLVM::AllocaOp::getPromotableSlots() {
647 if (!getOperation()->getBlock()->isEntryBlock())
648 return {};
649
650 return {MemorySlot{getResult(), getElemType()}};
651}
652
653Value LLVM::AllocaOp::getDefaultValue(const MemorySlot &slot,
654 OpBuilder &builder) {
655 return LLVM::UndefOp::create(builder, getLoc(), slot.valueType);
656}
657
658void LLVM::AllocaOp::handleBlockArgument(const MemorySlot &slot,
659 BlockArgument argument,
660 OpBuilder &builder) {
661 for (Operation *user : getOperation()->getUsers())
662 if (auto declareOp = llvm::dyn_cast<LLVM::DbgDeclareOp>(user))
663 LLVM::DbgValueOp::create(builder, declareOp.getLoc(), argument,
664 declareOp.getVarInfo(),
665 declareOp.getLocationExpr());
666}
667
668std::optional<PromotableAllocationOpInterface>
669LLVM::AllocaOp::handlePromotionComplete(const MemorySlot &slot,
670 Value defaultValue,
671 OpBuilder &builder) {
672 if (defaultValue && defaultValue.use_empty())
673 defaultValue.getDefiningOp()->erase();
674 this->erase();
675 return std::nullopt;
676}
677
678SmallVector<DestructurableMemorySlot> LLVM::AllocaOp::getDestructurableSlots() {
679 if (!mlir::matchPattern(getArraySize(), m_One()))
680 return {};
681
682 auto destructurable = dyn_cast<DestructurableTypeInterface>(getElemType());
683 if (!destructurable)
684 return {};
685
686 std::optional<DenseMap<Attribute, Type>> destructuredType =
687 destructurable.getSubelementIndexMap();
688 if (!destructuredType)
689 return {};
690
691 return {DestructurableMemorySlot{{getResult(), getElemType()},
692 *destructuredType}};
693}
694
695DenseMap<Attribute, MemorySlot> LLVM::AllocaOp::destructure(
696 const DestructurableMemorySlot &slot,
697 const SmallPtrSetImpl<Attribute> &usedIndices, OpBuilder &builder,
699 assert(slot.ptr == getResult());
700 builder.setInsertionPointAfter(*this);
701
702 auto destructurableType = cast<DestructurableTypeInterface>(getElemType());
704 // Iterate subelements in their original type order to produce allocas in a
705 // deterministic, readable order (matching appearance in the source type).
706 Type i32 = IntegerType::get(getContext(), 32);
707 for (size_t i = 0; i < slot.subelementTypes.size(); i++) {
708 Attribute index = IntegerAttr::get(i32, i);
709 if (!usedIndices.contains(index))
710 continue;
711 Type elemType = destructurableType.getTypeAtIndex(index);
712 assert(elemType && "used index must exist");
713 auto subAlloca = LLVM::AllocaOp::create(
714 builder, getLoc(), LLVM::LLVMPointerType::get(getContext()), elemType,
715 getArraySize());
716 newAllocators.push_back(subAlloca);
717 slotMap.try_emplace<MemorySlot>(index, {subAlloca.getResult(), elemType});
718 }
719
720 return slotMap;
721}
722
723std::optional<DestructurableAllocationOpInterface>
724LLVM::AllocaOp::handleDestructuringComplete(
725 const DestructurableMemorySlot &slot, OpBuilder &builder) {
726 assert(slot.ptr == getResult());
727 this->erase();
728 return std::nullopt;
729}
730
731//===----------------------------------------------------------------------===//
732// Interfaces for LoadOp/StoreOp
733//===----------------------------------------------------------------------===//
734
735bool LLVM::LoadOp::loadsFrom(const MemorySlot &slot) {
736 return getAddr() == slot.ptr;
737}
738
739bool LLVM::LoadOp::storesTo(const MemorySlot &slot) { return false; }
740
741Value LLVM::LoadOp::getStored(const MemorySlot &slot, OpBuilder &builder,
742 Value reachingDef, const DataLayout &dataLayout) {
743 llvm_unreachable("getStored should not be called on LoadOp");
744}
745
746bool LLVM::StoreOp::loadsFrom(const MemorySlot &slot) { return false; }
747
748bool LLVM::StoreOp::storesTo(const MemorySlot &slot) {
749 return getAddr() == slot.ptr;
750}
751
752/// Checks if `type` can be used in any kind of conversion sequences.
754 // Aggregate types are not bitcastable.
755 if (isa<LLVM::LLVMStructType, LLVM::LLVMArrayType>(type))
756 return false;
757
758 if (auto vectorType = dyn_cast<VectorType>(type)) {
759 // Vectors of pointers cannot be casted.
760 if (isa<LLVM::LLVMPointerType>(vectorType.getElementType()))
761 return false;
762 // Scalable types are not supported.
763 return !vectorType.isScalable();
764 }
765 return true;
766}
767
768/// Checks that `rhs` can be converted to `lhs` by a sequence of casts and
769/// truncations. Checks for narrowing or widening conversion compatibility
770/// depending on `narrowingConversion`.
771static bool areConversionCompatible(const DataLayout &layout, Type targetType,
772 Type srcType, bool narrowingConversion) {
773 if (targetType == srcType)
774 return true;
775
776 if (!isSupportedTypeForConversion(targetType) ||
778 return false;
779
780 uint64_t targetSize = layout.getTypeSize(targetType);
781 uint64_t srcSize = layout.getTypeSize(srcType);
782
783 // Pointer casts will only be sane when the bitsize of both pointer types is
784 // the same.
785 if (isa<LLVM::LLVMPointerType>(targetType) &&
786 isa<LLVM::LLVMPointerType>(srcType))
787 return targetSize == srcSize;
788
789 if (narrowingConversion)
790 return targetSize <= srcSize;
791 return targetSize >= srcSize;
792}
793
794/// Checks if `dataLayout` describes a little endian layout.
795static bool isBigEndian(const DataLayout &dataLayout) {
796 auto endiannessStr = dyn_cast_or_null<StringAttr>(dataLayout.getEndianness());
797 return endiannessStr && endiannessStr == "big";
798}
799
800/// Converts a value to an integer type of the same size.
801/// Assumes that the type can be converted.
803 const DataLayout &dataLayout) {
804 Type type = val.getType();
805 assert(isSupportedTypeForConversion(type) &&
806 "expected value to have a convertible type");
807
808 if (isa<IntegerType>(type))
809 return val;
810
811 uint64_t typeBitSize = dataLayout.getTypeSizeInBits(type);
812 IntegerType valueSizeInteger = builder.getIntegerType(typeBitSize);
813
814 if (isa<LLVM::LLVMPointerType>(type))
815 return builder.createOrFold<LLVM::PtrToIntOp>(loc, valueSizeInteger, val);
816 return builder.createOrFold<LLVM::BitcastOp>(loc, valueSizeInteger, val);
817}
818
819/// Converts a value with an integer type to `targetType`.
821 Value val, Type targetType) {
822 assert(isa<IntegerType>(val.getType()) &&
823 "expected value to have an integer type");
824 assert(isSupportedTypeForConversion(targetType) &&
825 "expected the target type to be supported for conversions");
826 if (val.getType() == targetType)
827 return val;
828 if (isa<LLVM::LLVMPointerType>(targetType))
829 return builder.createOrFold<LLVM::IntToPtrOp>(loc, targetType, val);
830 return builder.createOrFold<LLVM::BitcastOp>(loc, targetType, val);
831}
832
833/// Constructs operations that convert `srcValue` into a new value of type
834/// `targetType`. Assumes the types have the same bitsize.
836 Value srcValue, Type targetType,
837 const DataLayout &dataLayout) {
838 Type srcType = srcValue.getType();
839 assert(areConversionCompatible(dataLayout, targetType, srcType,
840 /*narrowingConversion=*/true) &&
841 "expected that the compatibility was checked before");
842
843 // Nothing has to be done if the types are already the same.
844 if (srcType == targetType)
845 return srcValue;
846
847 // In the special case of casting one pointer to another, we want to generate
848 // an address space cast. Bitcasts of pointers are not allowed and using
849 // pointer to integer conversions are not equivalent due to the loss of
850 // provenance.
851 if (isa<LLVM::LLVMPointerType>(targetType) &&
852 isa<LLVM::LLVMPointerType>(srcType))
853 return builder.createOrFold<LLVM::AddrSpaceCastOp>(loc, targetType,
854 srcValue);
855
856 // For all other castable types, casting through integers is necessary.
857 Value replacement = castToSameSizedInt(builder, loc, srcValue, dataLayout);
858 return castIntValueToSameSizedType(builder, loc, replacement, targetType);
859}
860
861/// Constructs operations that convert `srcValue` into a new value of type
862/// `targetType`. Performs bit-level extraction if the source type is larger
863/// than the target type. Assumes that this conversion is possible.
865 Value srcValue, Type targetType,
866 const DataLayout &dataLayout) {
867 // Get the types of the source and target values.
868 Type srcType = srcValue.getType();
869 assert(areConversionCompatible(dataLayout, targetType, srcType,
870 /*narrowingConversion=*/true) &&
871 "expected that the compatibility was checked before");
872
873 // Nothing has to be done if the types are already the same. This also
874 // avoids querying the bit size of scalable vector types below.
875 if (srcType == targetType)
876 return srcValue;
877
878 uint64_t srcTypeSize = dataLayout.getTypeSizeInBits(srcType);
879 uint64_t targetTypeSize = dataLayout.getTypeSizeInBits(targetType);
880 if (srcTypeSize == targetTypeSize)
881 return castSameSizedTypes(builder, loc, srcValue, targetType, dataLayout);
882
883 // First, cast the value to a same-sized integer type.
884 Value replacement = castToSameSizedInt(builder, loc, srcValue, dataLayout);
885
886 // Truncate the integer if the size of the target is less than the value.
887 if (isBigEndian(dataLayout)) {
888 uint64_t shiftAmount = srcTypeSize - targetTypeSize;
889 auto shiftConstant = LLVM::ConstantOp::create(
890 builder, loc, builder.getIntegerAttr(srcType, shiftAmount));
892 builder.createOrFold<LLVM::LShrOp>(loc, srcValue, shiftConstant);
893 }
894
895 replacement = LLVM::TruncOp::create(
896 builder, loc, builder.getIntegerType(targetTypeSize), replacement);
897
898 // Now cast the integer to the actual target type if required.
899 return castIntValueToSameSizedType(builder, loc, replacement, targetType);
900}
901
902/// Constructs operations that insert the bits of `srcValue` into the
903/// "beginning" of `reachingDef` (beginning is endianness dependent).
904/// Assumes that this conversion is possible.
906 Value srcValue, Value reachingDef,
907 const DataLayout &dataLayout) {
908
909 assert(areConversionCompatible(dataLayout, reachingDef.getType(),
910 srcValue.getType(),
911 /*narrowingConversion=*/false) &&
912 "expected that the compatibility was checked before");
913
914 // Nothing has to be done if the types are already the same. This also
915 // avoids querying the bit size of scalable vector types below.
916 if (srcValue.getType() == reachingDef.getType())
917 return srcValue;
918
919 uint64_t valueTypeSize = dataLayout.getTypeSizeInBits(srcValue.getType());
920 uint64_t slotTypeSize = dataLayout.getTypeSizeInBits(reachingDef.getType());
921 if (slotTypeSize == valueTypeSize)
922 return castSameSizedTypes(builder, loc, srcValue, reachingDef.getType(),
923 dataLayout);
924
925 // In the case where the store only overwrites parts of the memory,
926 // bit fiddling is required to construct the new value.
927
928 // First convert both values to integers of the same size.
929 Value defAsInt = castToSameSizedInt(builder, loc, reachingDef, dataLayout);
930 Value valueAsInt = castToSameSizedInt(builder, loc, srcValue, dataLayout);
931 // Extend the value to the size of the reaching definition.
932 valueAsInt =
933 builder.createOrFold<LLVM::ZExtOp>(loc, defAsInt.getType(), valueAsInt);
934 uint64_t sizeDifference = slotTypeSize - valueTypeSize;
935 if (isBigEndian(dataLayout)) {
936 // On big endian systems, a store to the base pointer overwrites the most
937 // significant bits. To accomodate for this, the stored value needs to be
938 // shifted into the according position.
939 Value bigEndianShift = LLVM::ConstantOp::create(
940 builder, loc,
941 builder.getIntegerAttr(defAsInt.getType(), sizeDifference));
942 valueAsInt =
943 builder.createOrFold<LLVM::ShlOp>(loc, valueAsInt, bigEndianShift);
944 }
945
946 // Construct the mask that is used to erase the bits that are overwritten by
947 // the store.
948 APInt maskValue;
949 if (isBigEndian(dataLayout)) {
950 // Build a mask that has the most significant bits set to zero.
951 // Note: This is the same as 2^sizeDifference - 1
952 maskValue = APInt::getAllOnes(sizeDifference).zext(slotTypeSize);
953 } else {
954 // Build a mask that has the least significant bits set to zero.
955 // Note: This is the same as -(2^valueTypeSize)
956 maskValue = APInt::getAllOnes(valueTypeSize).zext(slotTypeSize);
957 maskValue.flipAllBits();
958 }
959
960 // Mask out the affected bits ...
961 Value mask = LLVM::ConstantOp::create(
962 builder, loc, builder.getIntegerAttr(defAsInt.getType(), maskValue));
963 Value masked = builder.createOrFold<LLVM::AndOp>(loc, defAsInt, mask);
964
965 // ... and combine the result with the new value.
966 Value combined = builder.createOrFold<LLVM::OrOp>(loc, masked, valueAsInt);
967
968 return castIntValueToSameSizedType(builder, loc, combined,
969 reachingDef.getType());
970}
971
972Value LLVM::StoreOp::getStored(const MemorySlot &slot, OpBuilder &builder,
973 Value reachingDef,
974 const DataLayout &dataLayout) {
975 assert(reachingDef && reachingDef.getType() == slot.valueType &&
976 "expected the reaching definition's type to match the slot's type");
977 return createInsertAndCast(builder, getLoc(), getValue(), reachingDef,
978 dataLayout);
979}
980
981bool LLVM::LoadOp::canUsesBeRemoved(
982 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
983 SmallVectorImpl<OpOperand *> &newBlockingUses,
984 const DataLayout &dataLayout) {
985 if (blockingUses.size() != 1)
986 return false;
987 Value blockingUse = (*blockingUses.begin())->get();
988 // If the blocking use is the slot ptr itself, there will be enough
989 // context to reconstruct the result of the load at removal time, so it can
990 // be removed (provided it is not volatile).
991 return blockingUse == slot.ptr && getAddr() == slot.ptr &&
992 areConversionCompatible(dataLayout, getResult().getType(),
993 slot.valueType,
994 /*narrowingConversion=*/true) &&
995 !getVolatile_();
996}
997
998DeletionKind LLVM::LoadOp::removeBlockingUses(
999 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
1000 OpBuilder &builder, Value reachingDefinition,
1001 const DataLayout &dataLayout) {
1002 // `canUsesBeRemoved` checked this blocking use must be the loaded slot
1003 // pointer.
1004 Value newResult = createExtractAndCast(builder, getLoc(), reachingDefinition,
1005 getResult().getType(), dataLayout);
1006 getResult().replaceAllUsesWith(newResult);
1007 return DeletionKind::Delete;
1008}
1009
1010bool LLVM::StoreOp::canUsesBeRemoved(
1011 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
1012 SmallVectorImpl<OpOperand *> &newBlockingUses,
1013 const DataLayout &dataLayout) {
1014 if (blockingUses.size() != 1)
1015 return false;
1016 Value blockingUse = (*blockingUses.begin())->get();
1017 // If the blocking use is the slot ptr itself, dropping the store is
1018 // fine, provided we are currently promoting its target value. Don't allow a
1019 // store OF the slot pointer, only INTO the slot pointer.
1020 return blockingUse == slot.ptr && getAddr() == slot.ptr &&
1021 getValue() != slot.ptr &&
1022 areConversionCompatible(dataLayout, slot.valueType,
1023 getValue().getType(),
1024 /*narrowingConversion=*/false) &&
1025 !getVolatile_();
1026}
1027
1028DeletionKind LLVM::StoreOp::removeBlockingUses(
1029 const MemorySlot &slot, const SmallPtrSetImpl<OpOperand *> &blockingUses,
1030 OpBuilder &builder, Value reachingDefinition,
1031 const DataLayout &dataLayout) {
1032 return DeletionKind::Delete;
1033}
1034
1035/// Checks if `slot` can be accessed through the provided access type.
1036static bool isValidAccessType(const MemorySlot &slot, Type accessType,
1037 const DataLayout &dataLayout) {
1038 return dataLayout.getTypeSize(accessType) <=
1039 dataLayout.getTypeSize(slot.valueType);
1040}
1041
1042LogicalResult LLVM::LoadOp::ensureOnlySafeAccesses(
1043 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1044 const DataLayout &dataLayout) {
1045 return success(getAddr() != slot.ptr ||
1046 isValidAccessType(slot, getType(), dataLayout));
1047}
1048
1049LogicalResult LLVM::StoreOp::ensureOnlySafeAccesses(
1050 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1051 const DataLayout &dataLayout) {
1052 return success(getAddr() != slot.ptr ||
1053 isValidAccessType(slot, getValue().getType(), dataLayout));
1054}
1055
1056/// Returns the subslot's type at the requested index.
1058 Attribute index) {
1059 auto subelementIndexMap =
1060 cast<DestructurableTypeInterface>(slot.valueType).getSubelementIndexMap();
1061 if (!subelementIndexMap)
1062 return {};
1063 assert(!subelementIndexMap->empty());
1064
1065 // Note: Returns a null-type when no entry was found.
1066 return subelementIndexMap->lookup(index);
1067}
1068
1069bool LLVM::LoadOp::canRewire(const DestructurableMemorySlot &slot,
1070 SmallPtrSetImpl<Attribute> &usedIndices,
1071 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1072 const DataLayout &dataLayout) {
1073 if (getVolatile_())
1074 return false;
1075
1076 // A load always accesses the first element of the destructured slot.
1077 auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0);
1078 Type subslotType = getTypeAtIndex(slot, index);
1079 if (!subslotType)
1080 return false;
1081
1082 // The access can only be replaced when the subslot is read within its bounds.
1083 if (dataLayout.getTypeSize(getType()) > dataLayout.getTypeSize(subslotType))
1084 return false;
1085
1086 usedIndices.insert(index);
1087 return true;
1088}
1089
1090DeletionKind LLVM::LoadOp::rewire(const DestructurableMemorySlot &slot,
1092 OpBuilder &builder,
1093 const DataLayout &dataLayout) {
1094 auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0);
1095 auto it = subslots.find(index);
1096 assert(it != subslots.end());
1097
1098 getAddrMutable().set(it->getSecond().ptr);
1099 return DeletionKind::Keep;
1100}
1101
1102bool LLVM::StoreOp::canRewire(const DestructurableMemorySlot &slot,
1103 SmallPtrSetImpl<Attribute> &usedIndices,
1104 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1105 const DataLayout &dataLayout) {
1106 if (getVolatile_())
1107 return false;
1108
1109 // Storing the pointer to memory cannot be dealt with.
1110 if (getValue() == slot.ptr)
1111 return false;
1112
1113 // A store always accesses the first element of the destructured slot.
1114 auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0);
1115 Type subslotType = getTypeAtIndex(slot, index);
1116 if (!subslotType)
1117 return false;
1118
1119 // The access can only be replaced when the subslot is read within its bounds.
1120 if (dataLayout.getTypeSize(getValue().getType()) >
1121 dataLayout.getTypeSize(subslotType))
1122 return false;
1123
1124 usedIndices.insert(index);
1125 return true;
1126}
1127
1128DeletionKind LLVM::StoreOp::rewire(const DestructurableMemorySlot &slot,
1130 OpBuilder &builder,
1131 const DataLayout &dataLayout) {
1132 auto index = IntegerAttr::get(IntegerType::get(getContext(), 32), 0);
1133 auto it = subslots.find(index);
1134 assert(it != subslots.end());
1135
1136 getAddrMutable().set(it->getSecond().ptr);
1137 return DeletionKind::Keep;
1138}
1139
1140//===----------------------------------------------------------------------===//
1141// Interfaces for discardable OPs
1142//===----------------------------------------------------------------------===//
1143
1144/// Conditions the deletion of the operation to the removal of all its uses.
1146 SmallVectorImpl<OpOperand *> &newBlockingUses) {
1147 for (Value result : op->getResults())
1148 for (OpOperand &use : result.getUses())
1149 newBlockingUses.push_back(&use);
1150 return true;
1151}
1152
1153//===----------------------------------------------------------------------===//
1154// Interfaces for GEPOp
1155//===----------------------------------------------------------------------===//
1156
1157static bool hasAllZeroIndices(LLVM::GEPOp gepOp) {
1158 return llvm::all_of(gepOp.getIndices(), [](auto index) {
1159 auto indexAttr = llvm::dyn_cast_if_present<IntegerAttr>(index);
1160 return indexAttr && indexAttr.getValue() == 0;
1161 });
1162}
1163
1164bool LLVM::GEPOp::canUsesBeRemoved(
1165 const SmallPtrSetImpl<OpOperand *> &blockingUses,
1166 SmallVectorImpl<OpOperand *> &newBlockingUses,
1167 const DataLayout &dataLayout) {
1168 // GEP can be removed as long as it is a no-op and its users can be removed.
1169 // `inrange` is only valid on constant GEP expressions, so an inrange GEP on
1170 // an alloca is illegal and we bail out.
1171 if (getInrangeAttr() || !hasAllZeroIndices(*this))
1172 return false;
1173 return forwardToUsers(*this, newBlockingUses);
1174}
1175
1176DeletionKind LLVM::GEPOp::removeBlockingUses(
1177 const SmallPtrSetImpl<OpOperand *> &blockingUses, OpBuilder &builder) {
1178 return DeletionKind::Delete;
1179}
1180
1181/// Returns the amount of bytes the provided GEP elements will offset the
1182/// pointer by. Returns nullopt if no constant offset could be computed.
1183static std::optional<uint64_t> gepToByteOffset(const DataLayout &dataLayout,
1184 LLVM::GEPOp gep) {
1185 // Collects all indices.
1187 for (auto index : gep.getIndices()) {
1188 auto constIndex = dyn_cast<IntegerAttr>(index);
1189 if (!constIndex)
1190 return {};
1191 int64_t gepIndex = constIndex.getInt();
1192 // Negative indices are not supported.
1193 if (gepIndex < 0)
1194 return {};
1195 indices.push_back(gepIndex);
1196 }
1197
1198 Type currentType = gep.getElemType();
1199 uint64_t offset = indices[0] * dataLayout.getTypeSize(currentType);
1200
1201 for (uint64_t index : llvm::drop_begin(indices)) {
1202 bool shouldCancel =
1203 TypeSwitch<Type, bool>(currentType)
1204 .Case([&](LLVM::LLVMArrayType arrayType) {
1205 offset +=
1206 index * dataLayout.getTypeSize(arrayType.getElementType());
1207 currentType = arrayType.getElementType();
1208 return false;
1209 })
1210 .Case([&](LLVM::LLVMStructType structType) {
1211 offset += structType.getElementOffset(dataLayout, index);
1212 currentType = structType.getBody()[index];
1213 return false;
1214 })
1215 .Default([&](Type type) {
1216 LDBG() << "[sroa] Unsupported type for offset computations"
1217 << type;
1218 return true;
1219 });
1220
1221 if (shouldCancel)
1222 return std::nullopt;
1223 }
1224
1225 return offset;
1226}
1227
1228namespace {
1229/// A struct that stores both the index into the aggregate type of the slot as
1230/// well as the corresponding byte offset in memory.
1231struct SubslotAccessInfo {
1232 /// The parent slot's index that the access falls into.
1233 uint32_t index;
1234 /// The offset into the subslot of the access.
1235 uint64_t subslotOffset;
1236};
1237} // namespace
1238
1239/// Computes subslot access information for an access into `slot` with the given
1240/// offset.
1241/// Returns nullopt when the offset is out-of-bounds or when the access is into
1242/// the padding of `slot`.
1243static std::optional<SubslotAccessInfo>
1245 const DataLayout &dataLayout, LLVM::GEPOp gep) {
1246 std::optional<uint64_t> offset = gepToByteOffset(dataLayout, gep);
1247 if (!offset)
1248 return {};
1249
1250 // Helper to check that a constant index is in the bounds of the GEP index
1251 // representation. LLVM dialects's GEP arguments have a limited bitwidth, thus
1252 // this additional check is necessary.
1253 auto isOutOfBoundsGEPIndex = [](uint64_t index) {
1254 return index >= (1 << LLVM::kGEPConstantBitWidth);
1255 };
1256
1257 Type type = slot.valueType;
1258 if (*offset >= dataLayout.getTypeSize(type))
1259 return {};
1261 .Case([&](LLVM::LLVMArrayType arrayType)
1262 -> std::optional<SubslotAccessInfo> {
1263 // Find which element of the array contains the offset.
1264 uint64_t elemSize = dataLayout.getTypeSize(arrayType.getElementType());
1265 uint64_t index = *offset / elemSize;
1266 if (isOutOfBoundsGEPIndex(index))
1267 return {};
1268 return SubslotAccessInfo{static_cast<uint32_t>(index),
1269 *offset - (index * elemSize)};
1270 })
1271 .Case([&](LLVM::LLVMStructType structType)
1272 -> std::optional<SubslotAccessInfo> {
1273 // Walk over the elements of the struct to find in which of
1274 // them the offset is.
1275 for (auto [index, elem] : llvm::enumerate(structType.getBody())) {
1276 uint64_t elemOffset = structType.getElementOffset(dataLayout, index);
1277 // If the offset is in padding, cancel the rewrite.
1278 if (*offset < elemOffset)
1279 return {};
1280
1281 if (*offset < elemOffset + dataLayout.getTypeSize(elem)) {
1282 if (isOutOfBoundsGEPIndex(index))
1283 return {};
1284 // The offset is within this element, stop iterating the
1285 // struct and return the index.
1286 return SubslotAccessInfo{static_cast<uint32_t>(index),
1287 *offset - elemOffset};
1288 }
1289 }
1290
1291 return {};
1292 });
1293}
1294
1295/// Constructs a byte array type of the given size.
1296static LLVM::LLVMArrayType getByteArrayType(MLIRContext *context,
1297 unsigned size) {
1298 auto byteType = IntegerType::get(context, 8);
1299 return LLVM::LLVMArrayType::get(context, byteType, size);
1300}
1301
1302LogicalResult LLVM::GEPOp::ensureOnlySafeAccesses(
1303 const MemorySlot &slot, SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1304 const DataLayout &dataLayout) {
1305 if (getBase() != slot.ptr)
1306 return success();
1307 std::optional<uint64_t> gepOffset = gepToByteOffset(dataLayout, *this);
1308 if (!gepOffset)
1309 return failure();
1310 uint64_t slotSize = dataLayout.getTypeSize(slot.valueType);
1311 // Check that the access is strictly inside the slot.
1312 if (*gepOffset >= slotSize)
1313 return failure();
1314 // Every access that remains in bounds of the remaining slot is considered
1315 // legal.
1316 mustBeSafelyUsed.emplace_back<MemorySlot>(
1317 {getRes(), getByteArrayType(getContext(), slotSize - *gepOffset)});
1318 return success();
1319}
1320
1321bool LLVM::GEPOp::canRewire(const DestructurableMemorySlot &slot,
1322 SmallPtrSetImpl<Attribute> &usedIndices,
1323 SmallVectorImpl<MemorySlot> &mustBeSafelyUsed,
1324 const DataLayout &dataLayout) {
1325 if (!isa<LLVM::LLVMPointerType>(getBase().getType()))
1326 return false;
1327
1328 if (getBase() != slot.ptr)
1329 return false;
1330 // `inrange` is only valid on constant GEP expressions, so an inrange GEP on
1331 // an alloca is illegal and SROA bails out.
1332 if (getInrangeAttr())
1333 return false;
1334 std::optional<SubslotAccessInfo> accessInfo =
1335 getSubslotAccessInfo(slot, dataLayout, *this);
1336 if (!accessInfo)
1337 return false;
1338 auto indexAttr =
1339 IntegerAttr::get(IntegerType::get(getContext(), 32), accessInfo->index);
1340 assert(slot.subelementTypes.contains(indexAttr));
1341 usedIndices.insert(indexAttr);
1342
1343 // The remainder of the subslot should be accesses in-bounds. Thus, we create
1344 // a dummy slot with the size of the remainder.
1345 Type subslotType = slot.subelementTypes.lookup(indexAttr);
1346 uint64_t slotSize = dataLayout.getTypeSize(subslotType);
1347 LLVM::LLVMArrayType remainingSlotType =
1348 getByteArrayType(getContext(), slotSize - accessInfo->subslotOffset);
1349 mustBeSafelyUsed.emplace_back<MemorySlot>({getRes(), remainingSlotType});
1350
1351 return true;
1352}
1353
1354DeletionKind LLVM::GEPOp::rewire(const DestructurableMemorySlot &slot,
1356 OpBuilder &builder,
1357 const DataLayout &dataLayout) {
1358 std::optional<SubslotAccessInfo> accessInfo =
1359 getSubslotAccessInfo(slot, dataLayout, *this);
1360 assert(accessInfo && "expected access info to be checked before");
1361 auto indexAttr =
1362 IntegerAttr::get(IntegerType::get(getContext(), 32), accessInfo->index);
1363 const MemorySlot &newSlot = subslots.at(indexAttr);
1364
1365 auto byteType = IntegerType::get(builder.getContext(), 8);
1366 auto newPtr = builder.createOrFold<LLVM::GEPOp>(
1367 getLoc(), getResult().getType(), byteType, newSlot.ptr,
1368 ArrayRef<GEPArg>(accessInfo->subslotOffset), getNoWrapFlags());
1369 getResult().replaceAllUsesWith(newPtr);
1370 return DeletionKind::Delete;
1371}
1372
1373SmallVector<Value> mlir::LLVM::AtomicCmpXchgOp::getAccessedOperands() {
1374 return {getPtr()};
1375}
1376
1377SmallVector<Value> mlir::LLVM::AtomicRMWOp::getAccessedOperands() {
1378 return {getPtr()};
1379}
1380
1381SmallVector<Value> mlir::LLVM::LoadOp::getAccessedOperands() {
1382 return {getAddr()};
1383}
1384
1385SmallVector<Value> mlir::LLVM::StoreOp::getAccessedOperands() {
1386 return {getAddr()};
1387}
1388
1389#define GET_OP_CLASSES
1390#include "mlir/Dialect/LLVMIR/LLVMMemoryOps.cpp.inc"
return success()
static Value getBase(Value v)
Looks through known "view-like" ops to find the base memref.
static LLVM::LLVMArrayType getByteArrayType(MLIRContext *context, unsigned size)
Constructs a byte array type of the given size.
static LogicalResult verifyAtomicMemOp(OpTy memOp, Type valueType, ArrayRef< AtomicOrdering > unsupportedOrderings)
Verifies the attributes and the type of atomic memory access operations.
static std::optional< SubslotAccessInfo > getSubslotAccessInfo(const DestructurableMemorySlot &slot, const DataLayout &dataLayout, LLVM::GEPOp gep)
Computes subslot access information for an access into slot with the given offset.
static Value castToSameSizedInt(OpBuilder &builder, Location loc, Value val, const DataLayout &dataLayout)
Converts a value to an integer type of the same size.
static Value castSameSizedTypes(OpBuilder &builder, Location loc, Value srcValue, Type targetType, const DataLayout &dataLayout)
Constructs operations that convert srcValue into a new value of type targetType.
static Type getTypeAtIndex(const DestructurableMemorySlot &slot, Attribute index)
Returns the subslot's type at the requested index.
static bool areConversionCompatible(const DataLayout &layout, Type targetType, Type srcType, bool narrowingConversion)
Checks that rhs can be converted to lhs by a sequence of casts and truncations.
static bool isSupportedTypeForConversion(Type type)
Checks if type can be used in any kind of conversion sequences.
static Value createExtractAndCast(OpBuilder &builder, Location loc, Value srcValue, Type targetType, const DataLayout &dataLayout)
Constructs operations that convert srcValue into a new value of type targetType.
static Value createInsertAndCast(OpBuilder &builder, Location loc, Value srcValue, Value reachingDef, const DataLayout &dataLayout)
Constructs operations that insert the bits of srcValue into the "beginning" of reachingDef (beginning...
static bool isBigEndian(const DataLayout &dataLayout)
Checks if dataLayout describes a little endian layout.
static std::optional< uint64_t > gepToByteOffset(const DataLayout &dataLayout, LLVM::GEPOp gep)
Returns the amount of bytes the provided GEP elements will offset the pointer by.
static void destructureIndices(Type currType, ArrayRef< GEPArg > indices, SmallVectorImpl< int32_t > &rawConstantIndices, SmallVectorImpl< Value > &dynamicIndices)
Destructures the 'indices' parameter into 'rawConstantIndices' and 'dynamicIndices',...
static bool hasAllZeroIndices(LLVM::GEPOp gepOp)
static bool isValidAccessType(const MemorySlot &slot, Type accessType, const DataLayout &dataLayout)
Checks if slot can be accessed through the provided access type.
static constexpr const char kElemTypeAttrName[]
static LogicalResult verifyStructIndices(Type baseGEPType, unsigned indexPos, GEPIndicesAdaptor< ValueRange > indices, function_ref< InFlightDiagnostic()> emitOpError)
For the given indices, check if they comply with baseGEPType, especially check against LLVMStructType...
static Value castIntValueToSameSizedType(OpBuilder &builder, Location loc, Value val, Type targetType)
Converts a value with an integer type to targetType.
b getContext())
*if copies could not be generated due to yet unimplemented cases *copyInPlacementStart and copyOutPlacementStart in copyPlacementBlock *specify the insertion points where the incoming copies and outgoing should be the output argument nBegin is set to its * replacement(set to `begin` if no invalidation happens). Since outgoing *copies could have been inserted at `end`
virtual OptionalParseResult parseOptionalInteger(APInt &result)=0
Parse an optional integer value from the stream.
virtual ParseResult parseCommaSeparatedList(Delimiter delimiter, function_ref< ParseResult()> parseElementFn, StringRef contextMessage=StringRef())=0
Parse a list of comma-separated items with an optional delimiter.
virtual ParseResult parseOptionalAttrDict(NamedAttrList &result)=0
Parse a named dictionary into 'result' if it is present.
virtual ParseResult parseOptionalKeyword(StringRef keyword)=0
Parse the given keyword if present.
MLIRContext * getContext() const
virtual InFlightDiagnostic emitError(SMLoc loc, const Twine &message={})=0
Emit a diagnostic at the specified location and return failure.
virtual SMLoc getCurrentLocation()=0
Get the location of the next token and store it into the argument.
virtual ParseResult parseColon()=0
Parse a : token.
virtual SMLoc getNameLoc() const =0
Return the location of the original name token.
virtual ParseResult parseType(Type &result)=0
Parse a type.
ParseResult parseKeyword(StringRef keyword)
Parse a given keyword.
Attributes are known-constant values of operations.
Definition Attributes.h:25
This class represents an argument of a Block.
Definition Value.h:308
DenseI32ArrayAttr getDenseI32ArrayAttr(ArrayRef< int32_t > values)
Definition Builders.cpp:171
IntegerAttr getIntegerAttr(Type type, int64_t value)
Definition Builders.cpp:237
IntegerType getI32Type()
Definition Builders.cpp:71
IntegerAttr getI64IntegerAttr(int64_t value)
Definition Builders.cpp:120
IntegerType getIntegerType(unsigned width)
Definition Builders.cpp:75
StringAttr getStringAttr(const Twine &bytes)
Definition Builders.cpp:271
MLIRContext * getContext() const
Definition Builders.h:56
The main mechanism for performing data layout queries.
static DataLayout closest(Operation *op)
Returns the layout of the closest parent operation carrying layout info.
std::optional< uint64_t > getTypeIndexBitwidth(Type t) const
Returns the bitwidth that should be used when performing index computations for the given pointer-lik...
llvm::TypeSize getTypeSize(Type t) const
Returns the size of the given type in the current scope.
llvm::TypeSize getTypeSizeInBits(Type t) const
Returns the size in bits of the given type in the current scope.
Attribute getEndianness() const
Returns the specified endianness.
This class represents a diagnostic that is inflight and set to be reported.
Class used for building a 'llvm.getelementptr'.
Class used for convenient access and iteration over GEP indices.
This class defines the main interface for locations in MLIR and acts as a non-nullable wrapper around...
Definition Location.h:76
MLIRContext is the top-level object for a collection of MLIR operations.
Definition MLIRContext.h:71
NamedAttrList is array of NamedAttributes that tracks whether it is sorted and does some basic work t...
void append(StringRef name, Attribute attr)
Add an attribute with the specified name.
The OpAsmParser has methods for interacting with the asm parser: parsing things from it,...
virtual ParseResult resolveOperand(const UnresolvedOperand &operand, Type type, SmallVectorImpl< Value > &result)=0
Resolve an operand to an SSA value, emitting an error on failure.
virtual ParseResult parseOperand(UnresolvedOperand &result, bool allowResultNumber=true)=0
Parse a single SSA value operand name along with a result number if allowResultNumber is true.
This is a pure-virtual base class that exposes the asmprinter hooks necessary to implement a custom p...
virtual void printOptionalAttrDict(ArrayRef< NamedAttribute > attrs, ArrayRef< StringRef > elidedAttrs={})=0
If the specified operation has attributes, print out an attribute dictionary with their values.
virtual void printOperand(Value value)=0
Print implementations for various things an operation contains.
This class helps build Operations.
Definition Builders.h:210
void createOrFold(SmallVectorImpl< Value > &results, Location location, Args &&...args)
Create an operation of specific op type at the current insertion point, and immediately try to fold i...
Definition Builders.h:528
void setInsertionPointAfter(Operation *op)
Sets the insertion point to the node after the specified operation, which will cause subsequent inser...
Definition Builders.h:415
This class represents a single result from folding an operation.
This class represents an operand of an operation.
Definition Value.h:256
This class implements the operand iterators for the Operation class.
Definition ValueRange.h:44
Operation is the basic unit of execution within MLIR.
Definition Operation.h:87
result_range getResults()
Definition Operation.h:442
void erase()
Remove this operation from its parent block and delete it.
This class implements Optional functionality for ParseResult.
ParseResult value() const
Access the internal ParseResult value.
bool has_value() const
Returns true if we contain a valid ParseResult value.
A special type of RewriterBase that coordinates the application of a rewrite pattern on the current I...
virtual void replaceOp(Operation *op, ValueRange newValues)
Replace the results of the given (original) operation with the specified list of values (replacements...
This class represents a specific instance of an effect.
Instances of the Type class are uniqued, have an immutable identifier and an optional mutable compone...
Definition Types.h:74
MLIRContext * getContext() const
Return the MLIRContext in which this type was uniqued.
Definition Types.cpp:35
This class provides an abstraction over the different types of ranges over Values.
Definition ValueRange.h:392
This class represents an instance of an SSA value in the MLIR system, representing a computable value...
Definition Value.h:96
bool use_empty() const
Returns true if this value has no uses.
Definition Value.h:208
Type getType() const
Return the type of this value.
Definition Value.h:105
Operation * getDefiningOp() const
If this value is the result of an operation, return the operation that defines it.
Definition Value.cpp:18
static DenseArrayAttrImpl get(MLIRContext *context, ArrayRef< int32_t > content)
bool isScalableVectorType(Type vectorType)
Returns whether a vector type is scalable or not.
Type extractVectorElementType(Type type)
Returns the elemental type of any LLVM-compatible vector type or self.
bool isCompatibleVectorType(Type type)
Returns true if the given type is a vector type compatible with the LLVM dialect.
bool forwardToUsers(Operation *op, SmallVectorImpl< OpOperand * > &newBlockingUses)
Conditions the deletion of the operation to the removal of all its uses.
constexpr int kGEPConstantBitWidth
Bit-width of a 'GEPConstantIndex' within GEPArg.
bool isTypeCompatibleWithAtomicOp(Type type, const DataLayout &dataLayout)
Returns true if the given type is supported by atomic operations.
mlir::LLVM::LLVMStructType getValAndBoolStructType(mlir::Type valType)
Returns an LLVM struct type that contains a value type and a boolean type.
bool isCompatibleFloatingPointType(Type type)
Returns true if the given type is a floating-point type compatible with the LLVM dialect.
mlir::ParseResult parseGEPIndices(mlir::OpAsmParser &parser, mlir::SmallVectorImpl< mlir::OpAsmParser::UnresolvedOperand > &indices, mlir::DenseI32ArrayAttr &rawConstantIndices)
void printGEPIndices(mlir::OpAsmPrinter &printer, mlir::LLVM::GEPOp gepOp, mlir::OperandRange indices, mlir::DenseI32ArrayAttr rawConstantIndices)
Operation::operand_range getIndices(Operation *op)
Get the indices that the given load/store operation is operating on.
Definition Utils.cpp:18
Include the generated interface declarations.
bool matchPattern(Value value, const Pattern &pattern)
Entry point for matching a pattern over a Value.
Definition Matchers.h:490
detail::constant_int_value_binder m_ConstantInt(IntegerAttr::ValueType *bind_value)
Matches a constant holding a scalar/vector/tensor integer (splat) and writes the integer value to bin...
Definition Matchers.h:527
Type getType(OpFoldResult ofr)
Returns the int type of the integer in ofr.
Definition Utils.cpp:311
detail::DenseArrayAttrImpl< int32_t > DenseI32ArrayAttr
detail::constant_int_predicate_matcher m_One()
Matches a constant scalar / vector splat / tensor splat integer one.
Definition Matchers.h:478
DeletionKind
Returned by operation promotion logic requesting the deletion of an operation.
@ Keep
Keep the operation after promotion.
@ Delete
Delete the operation after promotion.
llvm::TypeSwitch< T, ResultT > TypeSwitch
Definition LLVM.h:147
llvm::DenseMap< KeyT, ValueT, KeyInfoT, BucketT > DenseMap
Definition LLVM.h:123
llvm::function_ref< Fn > function_ref
Definition LLVM.h:155
Memory slot attached with information about its destructuring procedure.
DenseMap< Attribute, Type > subelementTypes
Maps an index within the memory slot to the corresponding subelement type.
Represents a slot in memory.
Type valueType
Type of the value contained in the slot.
Value ptr
Pointer to the memory slot, used by operations to refer to it.
This is the representation of an operand reference.
This represents an operation in an abstracted form, suitable for use with the builder APIs.