5550c82189
The MLIR classes Type/Attribute/Operation/Op/Value support cast/dyn_cast/isa/dyn_cast_or_null functionality through llvm's doCast functionality in addition to defining methods with the same name. This change begins the migration of uses of the method to the corresponding function call as has been decided as more consistent. Note that there still exist classes that only define methods directly, such as AffineExpr, and this does not include work currently to support a functional cast/isa call. Caveats include: - This clang-tidy script probably has more problems. - This only touches C++ code, so nothing that is being generated. Context: - https://mlir.llvm.org/deprecation/ at "Use the free function variants for dyn_cast/cast/isa/…" - Original discussion at https://discourse.llvm.org/t/preferred-casting-style-going-forward/68443 Implementation: This first patch was created with the following steps. The intention is to only do automated changes at first, so I waste less time if it's reverted, and so the first mass change is more clear as an example to other teams that will need to follow similar steps. Steps are described per line, as comments are removed by git: 0. Retrieve the change from the following to build clang-tidy with an additional check: https://github.com/llvm/llvm-project/compare/main...tpopp:llvm-project:tidy-cast-check 1. Build clang-tidy 2. Run clang-tidy over your entire codebase while disabling all checks and enabling the one relevant one. Run on all header files also. 3. Delete .inc files that were also modified, so the next build rebuilds them to a pure state. 4. Some changes have been deleted for the following reasons: - Some files had a variable also named cast - Some files had not included a header file that defines the cast functions - Some files are definitions of the classes that have the casting methods, so the code still refers to the method instead of the function without adding a prefix or removing the method declaration at the same time. ``` ninja -C $BUILD_DIR clang-tidy run-clang-tidy -clang-tidy-binary=$BUILD_DIR/bin/clang-tidy -checks='-*,misc-cast-functions'\ -header-filter=mlir/ mlir/* -fix rm -rf $BUILD_DIR/tools/mlir/**/*.inc git restore mlir/lib/IR mlir/lib/Dialect/DLTI/DLTI.cpp\ mlir/lib/Dialect/Complex/IR/ComplexDialect.cpp\ mlir/lib/**/IR/\ mlir/lib/Dialect/SparseTensor/Transforms/SparseVectorization.cpp\ mlir/lib/Dialect/Vector/Transforms/LowerVectorMultiReduction.cpp\ mlir/test/lib/Dialect/Test/TestTypes.cpp\ mlir/test/lib/Dialect/Transform/TestTransformDialectExtension.cpp\ mlir/test/lib/Dialect/Test/TestAttributes.cpp\ mlir/unittests/TableGen/EnumsGenTest.cpp\ mlir/test/python/lib/PythonTestCAPI.cpp\ mlir/include/mlir/IR/ ``` Differential Revision: https://reviews.llvm.org/D150123
599 lines
25 KiB
C++
599 lines
25 KiB
C++
//===- MemRefBuilder.cpp - Helper for LLVM MemRef equivalents -------------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "mlir/Conversion/LLVMCommon/MemRefBuilder.h"
|
|
#include "MemRefDescriptor.h"
|
|
#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
|
|
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
|
|
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
|
|
#include "mlir/IR/Builders.h"
|
|
#include "mlir/Support/MathExtras.h"
|
|
|
|
using namespace mlir;
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MemRefDescriptor implementation
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Construct a helper for the given descriptor value.
|
|
MemRefDescriptor::MemRefDescriptor(Value descriptor)
|
|
: StructBuilder(descriptor) {
|
|
assert(value != nullptr && "value cannot be null");
|
|
indexType = cast<LLVM::LLVMStructType>(value.getType())
|
|
.getBody()[kOffsetPosInMemRefDescriptor];
|
|
}
|
|
|
|
/// Builds IR creating an `undef` value of the descriptor type.
|
|
MemRefDescriptor MemRefDescriptor::undef(OpBuilder &builder, Location loc,
|
|
Type descriptorType) {
|
|
|
|
Value descriptor = builder.create<LLVM::UndefOp>(loc, descriptorType);
|
|
return MemRefDescriptor(descriptor);
|
|
}
|
|
|
|
/// Builds IR creating a MemRef descriptor that represents `type` and
|
|
/// populates it with static shape and stride information extracted from the
|
|
/// type.
|
|
MemRefDescriptor
|
|
MemRefDescriptor::fromStaticShape(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
MemRefType type, Value memory) {
|
|
return fromStaticShape(builder, loc, typeConverter, type, memory, memory);
|
|
}
|
|
|
|
MemRefDescriptor MemRefDescriptor::fromStaticShape(
|
|
OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter,
|
|
MemRefType type, Value memory, Value alignedMemory) {
|
|
assert(type.hasStaticShape() && "unexpected dynamic shape");
|
|
|
|
// Extract all strides and offsets and verify they are static.
|
|
auto [strides, offset] = getStridesAndOffset(type);
|
|
assert(!ShapedType::isDynamic(offset) &&
|
|
"expected static offset");
|
|
assert(!llvm::any_of(strides, ShapedType::isDynamic) &&
|
|
"expected static strides");
|
|
|
|
auto convertedType = typeConverter.convertType(type);
|
|
assert(convertedType && "unexpected failure in memref type conversion");
|
|
|
|
auto descr = MemRefDescriptor::undef(builder, loc, convertedType);
|
|
descr.setAllocatedPtr(builder, loc, memory);
|
|
descr.setAlignedPtr(builder, loc, alignedMemory);
|
|
descr.setConstantOffset(builder, loc, offset);
|
|
|
|
// Fill in sizes and strides
|
|
for (unsigned i = 0, e = type.getRank(); i != e; ++i) {
|
|
descr.setConstantSize(builder, loc, i, type.getDimSize(i));
|
|
descr.setConstantStride(builder, loc, i, strides[i]);
|
|
}
|
|
return descr;
|
|
}
|
|
|
|
/// Builds IR extracting the allocated pointer from the descriptor.
|
|
Value MemRefDescriptor::allocatedPtr(OpBuilder &builder, Location loc) {
|
|
return extractPtr(builder, loc, kAllocatedPtrPosInMemRefDescriptor);
|
|
}
|
|
|
|
/// Builds IR inserting the allocated pointer into the descriptor.
|
|
void MemRefDescriptor::setAllocatedPtr(OpBuilder &builder, Location loc,
|
|
Value ptr) {
|
|
setPtr(builder, loc, kAllocatedPtrPosInMemRefDescriptor, ptr);
|
|
}
|
|
|
|
/// Builds IR extracting the aligned pointer from the descriptor.
|
|
Value MemRefDescriptor::alignedPtr(OpBuilder &builder, Location loc) {
|
|
return extractPtr(builder, loc, kAlignedPtrPosInMemRefDescriptor);
|
|
}
|
|
|
|
/// Builds IR inserting the aligned pointer into the descriptor.
|
|
void MemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc,
|
|
Value ptr) {
|
|
setPtr(builder, loc, kAlignedPtrPosInMemRefDescriptor, ptr);
|
|
}
|
|
|
|
// Creates a constant Op producing a value of `resultType` from an index-typed
|
|
// integer attribute.
|
|
static Value createIndexAttrConstant(OpBuilder &builder, Location loc,
|
|
Type resultType, int64_t value) {
|
|
return builder.create<LLVM::ConstantOp>(loc, resultType,
|
|
builder.getIndexAttr(value));
|
|
}
|
|
|
|
/// Builds IR extracting the offset from the descriptor.
|
|
Value MemRefDescriptor::offset(OpBuilder &builder, Location loc) {
|
|
return builder.create<LLVM::ExtractValueOp>(loc, value,
|
|
kOffsetPosInMemRefDescriptor);
|
|
}
|
|
|
|
/// Builds IR inserting the offset into the descriptor.
|
|
void MemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
|
|
Value offset) {
|
|
value = builder.create<LLVM::InsertValueOp>(loc, value, offset,
|
|
kOffsetPosInMemRefDescriptor);
|
|
}
|
|
|
|
/// Builds IR inserting the offset into the descriptor.
|
|
void MemRefDescriptor::setConstantOffset(OpBuilder &builder, Location loc,
|
|
uint64_t offset) {
|
|
setOffset(builder, loc,
|
|
createIndexAttrConstant(builder, loc, indexType, offset));
|
|
}
|
|
|
|
/// Builds IR extracting the pos-th size from the descriptor.
|
|
Value MemRefDescriptor::size(OpBuilder &builder, Location loc, unsigned pos) {
|
|
return builder.create<LLVM::ExtractValueOp>(
|
|
loc, value, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
|
|
}
|
|
|
|
Value MemRefDescriptor::size(OpBuilder &builder, Location loc, Value pos,
|
|
int64_t rank) {
|
|
auto arrayTy = LLVM::LLVMArrayType::get(indexType, rank);
|
|
|
|
LLVM::LLVMPointerType indexPtrTy;
|
|
LLVM::LLVMPointerType arrayPtrTy;
|
|
|
|
if (useOpaquePointers()) {
|
|
arrayPtrTy = indexPtrTy = LLVM::LLVMPointerType::get(builder.getContext());
|
|
} else {
|
|
indexPtrTy = LLVM::LLVMPointerType::get(indexType);
|
|
arrayPtrTy = LLVM::LLVMPointerType::get(arrayTy);
|
|
}
|
|
|
|
// Copy size values to stack-allocated memory.
|
|
auto one = createIndexAttrConstant(builder, loc, indexType, 1);
|
|
auto sizes = builder.create<LLVM::ExtractValueOp>(
|
|
loc, value, llvm::ArrayRef<int64_t>({kSizePosInMemRefDescriptor}));
|
|
auto sizesPtr = builder.create<LLVM::AllocaOp>(loc, arrayPtrTy, arrayTy, one,
|
|
/*alignment=*/0);
|
|
builder.create<LLVM::StoreOp>(loc, sizes, sizesPtr);
|
|
|
|
// Load an return size value of interest.
|
|
auto resultPtr = builder.create<LLVM::GEPOp>(
|
|
loc, indexPtrTy, arrayTy, sizesPtr, ArrayRef<LLVM::GEPArg>{0, pos});
|
|
return builder.create<LLVM::LoadOp>(loc, indexType, resultPtr);
|
|
}
|
|
|
|
/// Builds IR inserting the pos-th size into the descriptor
|
|
void MemRefDescriptor::setSize(OpBuilder &builder, Location loc, unsigned pos,
|
|
Value size) {
|
|
value = builder.create<LLVM::InsertValueOp>(
|
|
loc, value, size, ArrayRef<int64_t>({kSizePosInMemRefDescriptor, pos}));
|
|
}
|
|
|
|
void MemRefDescriptor::setConstantSize(OpBuilder &builder, Location loc,
|
|
unsigned pos, uint64_t size) {
|
|
setSize(builder, loc, pos,
|
|
createIndexAttrConstant(builder, loc, indexType, size));
|
|
}
|
|
|
|
/// Builds IR extracting the pos-th stride from the descriptor.
|
|
Value MemRefDescriptor::stride(OpBuilder &builder, Location loc, unsigned pos) {
|
|
return builder.create<LLVM::ExtractValueOp>(
|
|
loc, value, ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
|
|
}
|
|
|
|
/// Builds IR inserting the pos-th stride into the descriptor
|
|
void MemRefDescriptor::setStride(OpBuilder &builder, Location loc, unsigned pos,
|
|
Value stride) {
|
|
value = builder.create<LLVM::InsertValueOp>(
|
|
loc, value, stride,
|
|
ArrayRef<int64_t>({kStridePosInMemRefDescriptor, pos}));
|
|
}
|
|
|
|
void MemRefDescriptor::setConstantStride(OpBuilder &builder, Location loc,
|
|
unsigned pos, uint64_t stride) {
|
|
setStride(builder, loc, pos,
|
|
createIndexAttrConstant(builder, loc, indexType, stride));
|
|
}
|
|
|
|
LLVM::LLVMPointerType MemRefDescriptor::getElementPtrType() {
|
|
return cast<LLVM::LLVMPointerType>(
|
|
cast<LLVM::LLVMStructType>(value.getType())
|
|
.getBody()[kAlignedPtrPosInMemRefDescriptor]);
|
|
}
|
|
|
|
Value MemRefDescriptor::bufferPtr(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &converter,
|
|
MemRefType type) {
|
|
// When we convert to LLVM, the input memref must have been normalized
|
|
// beforehand. Hence, this call is guaranteed to work.
|
|
auto [strides, offsetCst] = getStridesAndOffset(type);
|
|
|
|
Value ptr = alignedPtr(builder, loc);
|
|
// Skip if offset is zero.
|
|
if (offsetCst != 0) {
|
|
Type indexType = converter.getIndexType();
|
|
Value offsetVal =
|
|
ShapedType::isDynamic(offsetCst)
|
|
? offset(builder, loc)
|
|
: createIndexAttrConstant(builder, loc, indexType, offsetCst);
|
|
Type elementType = converter.convertType(type.getElementType());
|
|
ptr = builder.create<LLVM::GEPOp>(loc, ptr.getType(), elementType, ptr,
|
|
offsetVal);
|
|
}
|
|
return ptr;
|
|
}
|
|
|
|
/// Creates a MemRef descriptor structure from a list of individual values
|
|
/// composing that descriptor, in the following order:
|
|
/// - allocated pointer;
|
|
/// - aligned pointer;
|
|
/// - offset;
|
|
/// - <rank> sizes;
|
|
/// - <rank> shapes;
|
|
/// where <rank> is the MemRef rank as provided in `type`.
|
|
Value MemRefDescriptor::pack(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &converter, MemRefType type,
|
|
ValueRange values) {
|
|
Type llvmType = converter.convertType(type);
|
|
auto d = MemRefDescriptor::undef(builder, loc, llvmType);
|
|
|
|
d.setAllocatedPtr(builder, loc, values[kAllocatedPtrPosInMemRefDescriptor]);
|
|
d.setAlignedPtr(builder, loc, values[kAlignedPtrPosInMemRefDescriptor]);
|
|
d.setOffset(builder, loc, values[kOffsetPosInMemRefDescriptor]);
|
|
|
|
int64_t rank = type.getRank();
|
|
for (unsigned i = 0; i < rank; ++i) {
|
|
d.setSize(builder, loc, i, values[kSizePosInMemRefDescriptor + i]);
|
|
d.setStride(builder, loc, i, values[kSizePosInMemRefDescriptor + rank + i]);
|
|
}
|
|
|
|
return d;
|
|
}
|
|
|
|
/// Builds IR extracting individual elements of a MemRef descriptor structure
|
|
/// and returning them as `results` list.
|
|
void MemRefDescriptor::unpack(OpBuilder &builder, Location loc, Value packed,
|
|
MemRefType type,
|
|
SmallVectorImpl<Value> &results) {
|
|
int64_t rank = type.getRank();
|
|
results.reserve(results.size() + getNumUnpackedValues(type));
|
|
|
|
MemRefDescriptor d(packed);
|
|
results.push_back(d.allocatedPtr(builder, loc));
|
|
results.push_back(d.alignedPtr(builder, loc));
|
|
results.push_back(d.offset(builder, loc));
|
|
for (int64_t i = 0; i < rank; ++i)
|
|
results.push_back(d.size(builder, loc, i));
|
|
for (int64_t i = 0; i < rank; ++i)
|
|
results.push_back(d.stride(builder, loc, i));
|
|
}
|
|
|
|
/// Returns the number of non-aggregate values that would be produced by
|
|
/// `unpack`.
|
|
unsigned MemRefDescriptor::getNumUnpackedValues(MemRefType type) {
|
|
// Two pointers, offset, <rank> sizes, <rank> shapes.
|
|
return 3 + 2 * type.getRank();
|
|
}
|
|
|
|
bool MemRefDescriptor::useOpaquePointers() {
|
|
return getElementPtrType().isOpaque();
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// MemRefDescriptorView implementation.
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
MemRefDescriptorView::MemRefDescriptorView(ValueRange range)
|
|
: rank((range.size() - kSizePosInMemRefDescriptor) / 2), elements(range) {}
|
|
|
|
Value MemRefDescriptorView::allocatedPtr() {
|
|
return elements[kAllocatedPtrPosInMemRefDescriptor];
|
|
}
|
|
|
|
Value MemRefDescriptorView::alignedPtr() {
|
|
return elements[kAlignedPtrPosInMemRefDescriptor];
|
|
}
|
|
|
|
Value MemRefDescriptorView::offset() {
|
|
return elements[kOffsetPosInMemRefDescriptor];
|
|
}
|
|
|
|
Value MemRefDescriptorView::size(unsigned pos) {
|
|
return elements[kSizePosInMemRefDescriptor + pos];
|
|
}
|
|
|
|
Value MemRefDescriptorView::stride(unsigned pos) {
|
|
return elements[kSizePosInMemRefDescriptor + rank + pos];
|
|
}
|
|
|
|
//===----------------------------------------------------------------------===//
|
|
// UnrankedMemRefDescriptor implementation
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
/// Construct a helper for the given descriptor value.
|
|
UnrankedMemRefDescriptor::UnrankedMemRefDescriptor(Value descriptor)
|
|
: StructBuilder(descriptor) {}
|
|
|
|
/// Builds IR creating an `undef` value of the descriptor type.
|
|
UnrankedMemRefDescriptor UnrankedMemRefDescriptor::undef(OpBuilder &builder,
|
|
Location loc,
|
|
Type descriptorType) {
|
|
Value descriptor = builder.create<LLVM::UndefOp>(loc, descriptorType);
|
|
return UnrankedMemRefDescriptor(descriptor);
|
|
}
|
|
Value UnrankedMemRefDescriptor::rank(OpBuilder &builder, Location loc) const {
|
|
return extractPtr(builder, loc, kRankInUnrankedMemRefDescriptor);
|
|
}
|
|
void UnrankedMemRefDescriptor::setRank(OpBuilder &builder, Location loc,
|
|
Value v) {
|
|
setPtr(builder, loc, kRankInUnrankedMemRefDescriptor, v);
|
|
}
|
|
Value UnrankedMemRefDescriptor::memRefDescPtr(OpBuilder &builder,
|
|
Location loc) const {
|
|
return extractPtr(builder, loc, kPtrInUnrankedMemRefDescriptor);
|
|
}
|
|
void UnrankedMemRefDescriptor::setMemRefDescPtr(OpBuilder &builder,
|
|
Location loc, Value v) {
|
|
setPtr(builder, loc, kPtrInUnrankedMemRefDescriptor, v);
|
|
}
|
|
|
|
/// Builds IR populating an unranked MemRef descriptor structure from a list
|
|
/// of individual constituent values in the following order:
|
|
/// - rank of the memref;
|
|
/// - pointer to the memref descriptor.
|
|
Value UnrankedMemRefDescriptor::pack(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &converter,
|
|
UnrankedMemRefType type,
|
|
ValueRange values) {
|
|
Type llvmType = converter.convertType(type);
|
|
auto d = UnrankedMemRefDescriptor::undef(builder, loc, llvmType);
|
|
|
|
d.setRank(builder, loc, values[kRankInUnrankedMemRefDescriptor]);
|
|
d.setMemRefDescPtr(builder, loc, values[kPtrInUnrankedMemRefDescriptor]);
|
|
return d;
|
|
}
|
|
|
|
/// Builds IR extracting individual elements that compose an unranked memref
|
|
/// descriptor and returns them as `results` list.
|
|
void UnrankedMemRefDescriptor::unpack(OpBuilder &builder, Location loc,
|
|
Value packed,
|
|
SmallVectorImpl<Value> &results) {
|
|
UnrankedMemRefDescriptor d(packed);
|
|
results.reserve(results.size() + 2);
|
|
results.push_back(d.rank(builder, loc));
|
|
results.push_back(d.memRefDescPtr(builder, loc));
|
|
}
|
|
|
|
void UnrankedMemRefDescriptor::computeSizes(
|
|
OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter,
|
|
ArrayRef<UnrankedMemRefDescriptor> values, ArrayRef<unsigned> addressSpaces,
|
|
SmallVectorImpl<Value> &sizes) {
|
|
if (values.empty())
|
|
return;
|
|
assert(values.size() == addressSpaces.size() &&
|
|
"must provide address space for each descriptor");
|
|
// Cache the index type.
|
|
Type indexType = typeConverter.getIndexType();
|
|
|
|
// Initialize shared constants.
|
|
Value one = createIndexAttrConstant(builder, loc, indexType, 1);
|
|
Value two = createIndexAttrConstant(builder, loc, indexType, 2);
|
|
Value indexSize =
|
|
createIndexAttrConstant(builder, loc, indexType,
|
|
ceilDiv(typeConverter.getIndexTypeBitwidth(), 8));
|
|
|
|
sizes.reserve(sizes.size() + values.size());
|
|
for (auto [desc, addressSpace] : llvm::zip(values, addressSpaces)) {
|
|
// Emit IR computing the memory necessary to store the descriptor. This
|
|
// assumes the descriptor to be
|
|
// { type*, type*, index, index[rank], index[rank] }
|
|
// and densely packed, so the total size is
|
|
// 2 * sizeof(pointer) + (1 + 2 * rank) * sizeof(index).
|
|
// TODO: consider including the actual size (including eventual padding due
|
|
// to data layout) into the unranked descriptor.
|
|
Value pointerSize = createIndexAttrConstant(
|
|
builder, loc, indexType,
|
|
ceilDiv(typeConverter.getPointerBitwidth(addressSpace), 8));
|
|
Value doublePointerSize =
|
|
builder.create<LLVM::MulOp>(loc, indexType, two, pointerSize);
|
|
|
|
// (1 + 2 * rank) * sizeof(index)
|
|
Value rank = desc.rank(builder, loc);
|
|
Value doubleRank = builder.create<LLVM::MulOp>(loc, indexType, two, rank);
|
|
Value doubleRankIncremented =
|
|
builder.create<LLVM::AddOp>(loc, indexType, doubleRank, one);
|
|
Value rankIndexSize = builder.create<LLVM::MulOp>(
|
|
loc, indexType, doubleRankIncremented, indexSize);
|
|
|
|
// Total allocation size.
|
|
Value allocationSize = builder.create<LLVM::AddOp>(
|
|
loc, indexType, doublePointerSize, rankIndexSize);
|
|
sizes.push_back(allocationSize);
|
|
}
|
|
}
|
|
|
|
Value UnrankedMemRefDescriptor::allocatedPtr(
|
|
OpBuilder &builder, Location loc, Value memRefDescPtr,
|
|
LLVM::LLVMPointerType elemPtrType) {
|
|
|
|
Value elementPtrPtr;
|
|
if (elemPtrType.isOpaque())
|
|
elementPtrPtr = memRefDescPtr;
|
|
else
|
|
elementPtrPtr = builder.create<LLVM::BitcastOp>(
|
|
loc, LLVM::LLVMPointerType::get(elemPtrType), memRefDescPtr);
|
|
|
|
return builder.create<LLVM::LoadOp>(loc, elemPtrType, elementPtrPtr);
|
|
}
|
|
|
|
void UnrankedMemRefDescriptor::setAllocatedPtr(
|
|
OpBuilder &builder, Location loc, Value memRefDescPtr,
|
|
LLVM::LLVMPointerType elemPtrType, Value allocatedPtr) {
|
|
Value elementPtrPtr;
|
|
if (elemPtrType.isOpaque())
|
|
elementPtrPtr = memRefDescPtr;
|
|
else
|
|
elementPtrPtr = builder.create<LLVM::BitcastOp>(
|
|
loc, LLVM::LLVMPointerType::get(elemPtrType), memRefDescPtr);
|
|
|
|
builder.create<LLVM::StoreOp>(loc, allocatedPtr, elementPtrPtr);
|
|
}
|
|
|
|
static std::pair<Value, Type>
|
|
castToElemPtrPtr(OpBuilder &builder, Location loc, Value memRefDescPtr,
|
|
LLVM::LLVMPointerType elemPtrType) {
|
|
Value elementPtrPtr;
|
|
Type elemPtrPtrType;
|
|
if (elemPtrType.isOpaque()) {
|
|
elementPtrPtr = memRefDescPtr;
|
|
elemPtrPtrType = LLVM::LLVMPointerType::get(builder.getContext());
|
|
} else {
|
|
elemPtrPtrType = LLVM::LLVMPointerType::get(elemPtrType);
|
|
elementPtrPtr =
|
|
builder.create<LLVM::BitcastOp>(loc, elemPtrPtrType, memRefDescPtr);
|
|
}
|
|
return {elementPtrPtr, elemPtrPtrType};
|
|
}
|
|
|
|
Value UnrankedMemRefDescriptor::alignedPtr(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value memRefDescPtr,
|
|
LLVM::LLVMPointerType elemPtrType) {
|
|
auto [elementPtrPtr, elemPtrPtrType] =
|
|
castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
|
|
|
|
Value alignedGep =
|
|
builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
|
|
elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
|
|
return builder.create<LLVM::LoadOp>(loc, elemPtrType, alignedGep);
|
|
}
|
|
|
|
void UnrankedMemRefDescriptor::setAlignedPtr(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value memRefDescPtr,
|
|
LLVM::LLVMPointerType elemPtrType,
|
|
Value alignedPtr) {
|
|
auto [elementPtrPtr, elemPtrPtrType] =
|
|
castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
|
|
|
|
Value alignedGep =
|
|
builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
|
|
elementPtrPtr, ArrayRef<LLVM::GEPArg>{1});
|
|
builder.create<LLVM::StoreOp>(loc, alignedPtr, alignedGep);
|
|
}
|
|
|
|
Value UnrankedMemRefDescriptor::offsetBasePtr(
|
|
OpBuilder &builder, Location loc, LLVMTypeConverter &typeConverter,
|
|
Value memRefDescPtr, LLVM::LLVMPointerType elemPtrType) {
|
|
auto [elementPtrPtr, elemPtrPtrType] =
|
|
castToElemPtrPtr(builder, loc, memRefDescPtr, elemPtrType);
|
|
|
|
Value offsetGep =
|
|
builder.create<LLVM::GEPOp>(loc, elemPtrPtrType, elemPtrType,
|
|
elementPtrPtr, ArrayRef<LLVM::GEPArg>{2});
|
|
|
|
if (!elemPtrType.isOpaque()) {
|
|
offsetGep = builder.create<LLVM::BitcastOp>(
|
|
loc, LLVM::LLVMPointerType::get(typeConverter.getIndexType()),
|
|
offsetGep);
|
|
}
|
|
return offsetGep;
|
|
}
|
|
|
|
Value UnrankedMemRefDescriptor::offset(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value memRefDescPtr,
|
|
LLVM::LLVMPointerType elemPtrType) {
|
|
Value offsetPtr =
|
|
offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
|
|
return builder.create<LLVM::LoadOp>(loc, typeConverter.getIndexType(),
|
|
offsetPtr);
|
|
}
|
|
|
|
void UnrankedMemRefDescriptor::setOffset(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value memRefDescPtr,
|
|
LLVM::LLVMPointerType elemPtrType,
|
|
Value offset) {
|
|
Value offsetPtr =
|
|
offsetBasePtr(builder, loc, typeConverter, memRefDescPtr, elemPtrType);
|
|
builder.create<LLVM::StoreOp>(loc, offset, offsetPtr);
|
|
}
|
|
|
|
Value UnrankedMemRefDescriptor::sizeBasePtr(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value memRefDescPtr,
|
|
LLVM::LLVMPointerType elemPtrType) {
|
|
Type indexTy = typeConverter.getIndexType();
|
|
Type structTy = LLVM::LLVMStructType::getLiteral(
|
|
indexTy.getContext(), {elemPtrType, elemPtrType, indexTy, indexTy});
|
|
Value structPtr;
|
|
if (elemPtrType.isOpaque()) {
|
|
structPtr = memRefDescPtr;
|
|
} else {
|
|
Type structPtrTy = LLVM::LLVMPointerType::get(structTy);
|
|
structPtr =
|
|
builder.create<LLVM::BitcastOp>(loc, structPtrTy, memRefDescPtr);
|
|
}
|
|
|
|
auto resultType = elemPtrType.isOpaque()
|
|
? LLVM::LLVMPointerType::get(indexTy.getContext())
|
|
: LLVM::LLVMPointerType::get(indexTy);
|
|
return builder.create<LLVM::GEPOp>(loc, resultType, structTy, structPtr,
|
|
ArrayRef<LLVM::GEPArg>{0, 3});
|
|
}
|
|
|
|
Value UnrankedMemRefDescriptor::size(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value sizeBasePtr, Value index) {
|
|
|
|
Type indexTy = typeConverter.getIndexType();
|
|
Type indexPtrTy = typeConverter.getPointerType(indexTy);
|
|
|
|
Value sizeStoreGep =
|
|
builder.create<LLVM::GEPOp>(loc, indexPtrTy, indexTy, sizeBasePtr, index);
|
|
return builder.create<LLVM::LoadOp>(loc, indexTy, sizeStoreGep);
|
|
}
|
|
|
|
void UnrankedMemRefDescriptor::setSize(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value sizeBasePtr, Value index,
|
|
Value size) {
|
|
Type indexTy = typeConverter.getIndexType();
|
|
Type indexPtrTy = typeConverter.getPointerType(indexTy);
|
|
|
|
Value sizeStoreGep =
|
|
builder.create<LLVM::GEPOp>(loc, indexPtrTy, indexTy, sizeBasePtr, index);
|
|
builder.create<LLVM::StoreOp>(loc, size, sizeStoreGep);
|
|
}
|
|
|
|
Value UnrankedMemRefDescriptor::strideBasePtr(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value sizeBasePtr, Value rank) {
|
|
Type indexTy = typeConverter.getIndexType();
|
|
Type indexPtrTy = typeConverter.getPointerType(indexTy);
|
|
|
|
return builder.create<LLVM::GEPOp>(loc, indexPtrTy, indexTy, sizeBasePtr,
|
|
rank);
|
|
}
|
|
|
|
Value UnrankedMemRefDescriptor::stride(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value strideBasePtr, Value index,
|
|
Value stride) {
|
|
Type indexTy = typeConverter.getIndexType();
|
|
Type indexPtrTy = typeConverter.getPointerType(indexTy);
|
|
|
|
Value strideStoreGep = builder.create<LLVM::GEPOp>(loc, indexPtrTy, indexTy,
|
|
strideBasePtr, index);
|
|
return builder.create<LLVM::LoadOp>(loc, indexTy, strideStoreGep);
|
|
}
|
|
|
|
void UnrankedMemRefDescriptor::setStride(OpBuilder &builder, Location loc,
|
|
LLVMTypeConverter &typeConverter,
|
|
Value strideBasePtr, Value index,
|
|
Value stride) {
|
|
Type indexTy = typeConverter.getIndexType();
|
|
Type indexPtrTy = typeConverter.getPointerType(indexTy);
|
|
|
|
Value strideStoreGep = builder.create<LLVM::GEPOp>(loc, indexPtrTy, indexTy,
|
|
strideBasePtr, index);
|
|
builder.create<LLVM::StoreOp>(loc, stride, strideStoreGep);
|
|
}
|