shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/clang/lib/CIR/Lowering/DirectToLLVM/LowerToLLVM.cpp
Andy Kaylor 511a7aacee
[CIR][NFC] Use tablegen to create CIRAttrToValue visitor declarations (#187607) 
This change introduces TableGen support for indicating CIR attributes
that require a CIRAttrToValue visitor, adds the new flag to all
attributes to which it applies, and replaces the explicit declarations
with the tablegen output.
2026-04-06 17:11:25 -07:00

4735 lines
190 KiB
C++
Raw Blame History

//====- LowerToLLVM.cpp - Lowering from CIR to LLVMIR ---------------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file implements lowering of CIR operations to LLVMIR.
//
//===----------------------------------------------------------------------===//
#include "LowerToLLVM.h"
#include <array>
#include <deque>
#include <optional>
#include "mlir/Conversion/LLVMCommon/TypeConverter.h"
#include "mlir/Dialect/DLTI/DLTI.h"
#include "mlir/Dialect/Func/IR/FuncOps.h"
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
#include "mlir/Dialect/OpenMP/Transforms/Passes.h"
#include "mlir/Dialect/Ptr/IR/MemorySpaceInterfaces.h"
#include "mlir/IR/BuiltinAttributes.h"
#include "mlir/IR/BuiltinDialect.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/Types.h"
#include "mlir/Pass/Pass.h"
#include "mlir/Pass/PassManager.h"
#include "mlir/Support/LLVM.h"
#include "mlir/Target/LLVMIR/Dialect/Builtin/BuiltinToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/LLVMIR/LLVMToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Dialect/OpenMP/OpenMPToLLVMIRTranslation.h"
#include "mlir/Target/LLVMIR/Export.h"
#include "mlir/Transforms/DialectConversion.h"
#include "clang/CIR/Dialect/IR/CIRAttrs.h"
#include "clang/CIR/Dialect/IR/CIRDialect.h"
#include "clang/CIR/Dialect/IR/CIRTypes.h"
#include "clang/CIR/Dialect/Passes.h"
#include "clang/CIR/LoweringHelpers.h"
#include "clang/CIR/MissingFeatures.h"
#include "clang/CIR/Passes.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/IR/Module.h"
#include "llvm/Support/Casting.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/TimeProfiler.h"
#include "llvm/Support/raw_ostream.h"
using namespace cir;
using namespace llvm;
namespace cir {
namespace direct {
//===----------------------------------------------------------------------===//
// Helper Methods
//===----------------------------------------------------------------------===//
namespace {
/// If the given type is a vector type, return the vector's element type.
/// Otherwise return the given type unchanged.
mlir::Type elementTypeIfVector(mlir::Type type) {
return llvm::TypeSwitch<mlir::Type, mlir::Type>(type)
.Case<cir::VectorType, mlir::VectorType>(
[](auto p) { return p.getElementType(); })
.Default([](mlir::Type p) { return p; });
}
} // namespace
/// Given a type convertor and a data layout, convert the given type to a type
/// that is suitable for memory operations. For example, this can be used to
/// lower cir.bool accesses to i8.
static mlir::Type convertTypeForMemory(const mlir::TypeConverter &converter,
mlir::DataLayout const &dataLayout,
mlir::Type type) {
// TODO(cir): Handle other types similarly to clang's codegen
// convertTypeForMemory
if (isa<cir::BoolType>(type)) {
return mlir::IntegerType::get(type.getContext(),
dataLayout.getTypeSizeInBits(type));
}
return converter.convertType(type);
}
static mlir::Value createIntCast(mlir::OpBuilder &bld, mlir::Value src,
mlir::IntegerType dstTy,
bool isSigned = false) {
mlir::Type srcTy = src.getType();
assert(mlir::isa<mlir::IntegerType>(srcTy));
unsigned srcWidth = mlir::cast<mlir::IntegerType>(srcTy).getWidth();
unsigned dstWidth = mlir::cast<mlir::IntegerType>(dstTy).getWidth();
mlir::Location loc = src.getLoc();
if (dstWidth > srcWidth && isSigned)
return mlir::LLVM::SExtOp::create(bld, loc, dstTy, src);
if (dstWidth > srcWidth)
return mlir::LLVM::ZExtOp::create(bld, loc, dstTy, src);
if (dstWidth < srcWidth)
return mlir::LLVM::TruncOp::create(bld, loc, dstTy, src);
return mlir::LLVM::BitcastOp::create(bld, loc, dstTy, src);
}
static mlir::LLVM::Visibility
lowerCIRVisibilityToLLVMVisibility(cir::VisibilityKind visibilityKind) {
switch (visibilityKind) {
case cir::VisibilityKind::Default:
return ::mlir::LLVM::Visibility::Default;
case cir::VisibilityKind::Hidden:
return ::mlir::LLVM::Visibility::Hidden;
case cir::VisibilityKind::Protected:
return ::mlir::LLVM::Visibility::Protected;
}
}
/// Emits the value from memory as expected by its users. Should be called when
/// the memory represetnation of a CIR type is not equal to its scalar
/// representation.
static mlir::Value emitFromMemory(mlir::ConversionPatternRewriter &rewriter,
mlir::DataLayout const &dataLayout,
cir::LoadOp op, mlir::Value value) {
// TODO(cir): Handle other types similarly to clang's codegen EmitFromMemory
if (auto boolTy = mlir::dyn_cast<cir::BoolType>(op.getType())) {
// Create a cast value from specified size in datalayout to i1
assert(value.getType().isInteger(dataLayout.getTypeSizeInBits(boolTy)));
return createIntCast(rewriter, value, rewriter.getI1Type());
}
return value;
}
/// Emits a value to memory with the expected scalar type. Should be called when
/// the memory represetnation of a CIR type is not equal to its scalar
/// representation.
static mlir::Value emitToMemory(mlir::ConversionPatternRewriter &rewriter,
mlir::DataLayout const &dataLayout,
mlir::Type origType, mlir::Value value) {
// TODO(cir): Handle other types similarly to clang's codegen EmitToMemory
if (auto boolTy = mlir::dyn_cast<cir::BoolType>(origType)) {
// Create zext of value from i1 to i8
mlir::IntegerType memType =
rewriter.getIntegerType(dataLayout.getTypeSizeInBits(boolTy));
return createIntCast(rewriter, value, memType);
}
return value;
}
mlir::LLVM::Linkage convertLinkage(cir::GlobalLinkageKind linkage) {
using CIR = cir::GlobalLinkageKind;
using LLVM = mlir::LLVM::Linkage;
switch (linkage) {
case CIR::AppendingLinkage:
return LLVM::Appending;
case CIR::AvailableExternallyLinkage:
return LLVM::AvailableExternally;
case CIR::CommonLinkage:
return LLVM::Common;
case CIR::ExternalLinkage:
return LLVM::External;
case CIR::ExternalWeakLinkage:
return LLVM::ExternWeak;
case CIR::InternalLinkage:
return LLVM::Internal;
case CIR::LinkOnceAnyLinkage:
return LLVM::Linkonce;
case CIR::LinkOnceODRLinkage:
return LLVM::LinkonceODR;
case CIR::PrivateLinkage:
return LLVM::Private;
case CIR::WeakAnyLinkage:
return LLVM::Weak;
case CIR::WeakODRLinkage:
return LLVM::WeakODR;
};
llvm_unreachable("Unknown CIR linkage type");
}
mlir::LogicalResult CIRToLLVMCopyOpLowering::matchAndRewrite(
cir::CopyOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
const mlir::Value length = mlir::LLVM::ConstantOp::create(
rewriter, op.getLoc(), rewriter.getI64Type(),
op.getCopySizeInBytes(layout));
assert(!cir::MissingFeatures::aggValueSlotVolatile());
rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(
op, adaptor.getDst(), adaptor.getSrc(), length, op.getIsVolatile());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMMemCpyOpLowering::matchAndRewrite(
cir::MemCpyOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::MemcpyOp>(
op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLen(),
/*isVolatile=*/false);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMMemMoveOpLowering::matchAndRewrite(
cir::MemMoveOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::MemmoveOp>(
op, adaptor.getDst(), adaptor.getSrc(), adaptor.getLen(),
/*isVolatile=*/false);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMMemSetOpLowering::matchAndRewrite(
cir::MemSetOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto memset = rewriter.replaceOpWithNewOp<mlir::LLVM::MemsetOp>(
op, adaptor.getDst(), adaptor.getVal(), adaptor.getLen(),
/*isVolatile=*/false);
if (op.getAlignmentAttr()) {
// Construct a list full of empty attributes.
llvm::SmallVector<mlir::Attribute> attrs{memset.getNumOperands(),
rewriter.getDictionaryAttr({})};
llvm::SmallVector<mlir::NamedAttribute> destAttrs;
destAttrs.push_back(
{mlir::LLVM::LLVMDialect::getAlignAttrName(), op.getAlignmentAttr()});
attrs[memset.odsIndex_dst] = rewriter.getDictionaryAttr(destAttrs);
auto arrayAttr = rewriter.getArrayAttr(attrs);
memset.setArgAttrsAttr(arrayAttr);
}
return mlir::success();
}
static mlir::Value getLLVMIntCast(mlir::ConversionPatternRewriter &rewriter,
mlir::Value llvmSrc, mlir::Type llvmDstIntTy,
bool isUnsigned, uint64_t cirSrcWidth,
uint64_t cirDstIntWidth) {
if (cirSrcWidth == cirDstIntWidth)
return llvmSrc;
auto loc = llvmSrc.getLoc();
if (cirSrcWidth < cirDstIntWidth) {
if (isUnsigned)
return mlir::LLVM::ZExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);
return mlir::LLVM::SExtOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);
}
// Otherwise truncate
return mlir::LLVM::TruncOp::create(rewriter, loc, llvmDstIntTy, llvmSrc);
}
class CIRAttrToValue {
public:
CIRAttrToValue(mlir::Operation *parentOp,
mlir::ConversionPatternRewriter &rewriter,
const mlir::TypeConverter *converter)
: parentOp(parentOp), rewriter(rewriter), converter(converter) {}
#define GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS
#include "clang/CIR/Dialect/IR/CIRLowering.inc"
#undef GET_CIR_ATTR_TO_VALUE_VISITOR_DECLS
private:
mlir::Operation *parentOp;
mlir::ConversionPatternRewriter &rewriter;
const mlir::TypeConverter *converter;
};
/// Switches on the type of attribute and calls the appropriate conversion.
mlir::Value lowerCirAttrAsValue(mlir::Operation *parentOp,
const mlir::Attribute attr,
mlir::ConversionPatternRewriter &rewriter,
const mlir::TypeConverter *converter) {
CIRAttrToValue valueConverter(parentOp, rewriter, converter);
mlir::Value value = valueConverter.visit(attr);
if (!value)
llvm_unreachable("unhandled attribute type");
return value;
}
void convertSideEffectForCall(mlir::Operation *callOp, bool isNothrow,
cir::SideEffect sideEffect,
mlir::LLVM::MemoryEffectsAttr &memoryEffect,
bool &noUnwind, bool &willReturn,
bool &noReturn) {
using mlir::LLVM::ModRefInfo;
switch (sideEffect) {
case cir::SideEffect::All:
memoryEffect = {};
noUnwind = isNothrow;
willReturn = false;
break;
case cir::SideEffect::Pure:
memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
callOp->getContext(), /*other=*/ModRefInfo::Ref,
/*argMem=*/ModRefInfo::Ref,
/*inaccessibleMem=*/ModRefInfo::Ref,
/*errnoMem=*/ModRefInfo::Ref,
/*targetMem0=*/ModRefInfo::Ref,
/*targetMem1=*/ModRefInfo::Ref);
noUnwind = true;
willReturn = true;
break;
case cir::SideEffect::Const:
memoryEffect = mlir::LLVM::MemoryEffectsAttr::get(
callOp->getContext(), /*other=*/ModRefInfo::NoModRef,
/*argMem=*/ModRefInfo::NoModRef,
/*inaccessibleMem=*/ModRefInfo::NoModRef,
/*errnoMem=*/ModRefInfo::NoModRef,
/*targetMem0=*/ModRefInfo::NoModRef,
/*targetMem1=*/ModRefInfo::NoModRef);
noUnwind = true;
willReturn = true;
break;
}
noReturn = callOp->hasAttr(CIRDialect::getNoReturnAttrName());
}
static mlir::LLVM::CallIntrinsicOp
createCallLLVMIntrinsicOp(mlir::ConversionPatternRewriter &rewriter,
mlir::Location loc, const llvm::Twine &intrinsicName,
mlir::Type resultTy, mlir::ValueRange operands) {
auto intrinsicNameAttr =
mlir::StringAttr::get(rewriter.getContext(), intrinsicName);
return mlir::LLVM::CallIntrinsicOp::create(rewriter, loc, resultTy,
intrinsicNameAttr, operands);
}
static mlir::LLVM::CallIntrinsicOp replaceOpWithCallLLVMIntrinsicOp(
mlir::ConversionPatternRewriter &rewriter, mlir::Operation *op,
const llvm::Twine &intrinsicName, mlir::Type resultTy,
mlir::ValueRange operands) {
mlir::LLVM::CallIntrinsicOp callIntrinOp = createCallLLVMIntrinsicOp(
rewriter, op->getLoc(), intrinsicName, resultTy, operands);
rewriter.replaceOp(op, callIntrinOp.getOperation());
return callIntrinOp;
}
mlir::LogicalResult CIRToLLVMLLVMIntrinsicCallOpLowering::matchAndRewrite(
cir::LLVMIntrinsicCallOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type llvmResTy =
getTypeConverter()->convertType(op->getResultTypes()[0]);
if (!llvmResTy)
return op.emitError("expected LLVM result type");
StringRef name = op.getIntrinsicName();
// Some LLVM intrinsics require ElementType attribute to be attached to
// the argument of pointer type. That prevents us from generating LLVM IR
// because from LLVM dialect, we have LLVM IR like the below which fails
// LLVM IR verification.
// %3 = call i64 @llvm.aarch64.ldxr.p0(ptr %2)
// The expected LLVM IR should be like
// %3 = call i64 @llvm.aarch64.ldxr.p0(ptr elementtype(i32) %2)
// TODO(cir): MLIR LLVM dialect should handle this part as CIR has no way
// to set LLVM IR attribute.
assert(!cir::MissingFeatures::intrinsicElementTypeSupport());
replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm." + name, llvmResTy,
adaptor.getOperands());
return mlir::success();
}
/// BoolAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::BoolAttr boolAttr) {
mlir::Location loc = parentOp->getLoc();
mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>());
mlir::Value boolVal = mlir::LLVM::ConstantOp::create(
rewriter, loc, converter->convertType(boolAttr.getType()),
boolAttr.getValue());
return emitToMemory(rewriter, layout, boolAttr.getType(), boolVal);
}
/// IntAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::IntAttr intAttr) {
mlir::Location loc = parentOp->getLoc();
return mlir::LLVM::ConstantOp::create(
rewriter, loc, converter->convertType(intAttr.getType()),
intAttr.getValue());
}
/// FPAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::FPAttr fltAttr) {
mlir::Location loc = parentOp->getLoc();
return mlir::LLVM::ConstantOp::create(
rewriter, loc, converter->convertType(fltAttr.getType()),
fltAttr.getValue());
}
/// ConstComplexAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstComplexAttr complexAttr) {
auto complexType = mlir::cast<cir::ComplexType>(complexAttr.getType());
mlir::Type complexElemTy = complexType.getElementType();
mlir::Type complexElemLLVMTy = converter->convertType(complexElemTy);
mlir::Attribute components[2];
if (const auto intType = mlir::dyn_cast<cir::IntType>(complexElemTy)) {
components[0] = rewriter.getIntegerAttr(
complexElemLLVMTy,
mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
components[1] = rewriter.getIntegerAttr(
complexElemLLVMTy,
mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
} else {
components[0] = rewriter.getFloatAttr(
complexElemLLVMTy,
mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
components[1] = rewriter.getFloatAttr(
complexElemLLVMTy,
mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
}
mlir::Location loc = parentOp->getLoc();
return mlir::LLVM::ConstantOp::create(
rewriter, loc, converter->convertType(complexAttr.getType()),
rewriter.getArrayAttr(components));
}
/// ConstPtrAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstPtrAttr ptrAttr) {
mlir::Location loc = parentOp->getLoc();
if (ptrAttr.isNullValue()) {
return mlir::LLVM::ZeroOp::create(
rewriter, loc, converter->convertType(ptrAttr.getType()));
}
mlir::DataLayout layout(parentOp->getParentOfType<mlir::ModuleOp>());
mlir::Value ptrVal = mlir::LLVM::ConstantOp::create(
rewriter, loc,
rewriter.getIntegerType(layout.getTypeSizeInBits(ptrAttr.getType())),
ptrAttr.getValue().getInt());
return mlir::LLVM::IntToPtrOp::create(
rewriter, loc, converter->convertType(ptrAttr.getType()), ptrVal);
}
// ConstArrayAttr visitor
mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstArrayAttr attr) {
mlir::Type llvmTy = converter->convertType(attr.getType());
mlir::Location loc = parentOp->getLoc();
mlir::Value result;
if (attr.hasTrailingZeros()) {
mlir::Type arrayTy = attr.getType();
result = mlir::LLVM::ZeroOp::create(rewriter, loc,
converter->convertType(arrayTy));
} else {
result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
}
// Iteratively lower each constant element of the array.
if (auto arrayAttr = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts())) {
for (auto [idx, elt] : llvm::enumerate(arrayAttr)) {
mlir::DataLayout dataLayout(parentOp->getParentOfType<mlir::ModuleOp>());
mlir::Value init = visit(elt);
result =
mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
}
} else if (auto strAttr = mlir::dyn_cast<mlir::StringAttr>(attr.getElts())) {
// TODO(cir): this diverges from traditional lowering. Normally the string
// would be a global constant that is memcopied.
auto arrayTy = mlir::dyn_cast<cir::ArrayType>(strAttr.getType());
assert(arrayTy && "String attribute must have an array type");
mlir::Type eltTy = arrayTy.getElementType();
for (auto [idx, elt] : llvm::enumerate(strAttr)) {
auto init = mlir::LLVM::ConstantOp::create(
rewriter, loc, converter->convertType(eltTy), elt);
result =
mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
}
} else {
llvm_unreachable("unexpected ConstArrayAttr elements");
}
return result;
}
/// ConstRecord visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstRecordAttr constRecord) {
const mlir::Type llvmTy = converter->convertType(constRecord.getType());
const mlir::Location loc = parentOp->getLoc();
mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
// Iteratively lower each constant element of the record.
for (auto [idx, elt] : llvm::enumerate(constRecord.getMembers())) {
mlir::Value init = visit(elt);
result =
mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
}
return result;
}
/// ConstVectorAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::ConstVectorAttr attr) {
const mlir::Type llvmTy = converter->convertType(attr.getType());
const mlir::Location loc = parentOp->getLoc();
SmallVector<mlir::Attribute> mlirValues;
for (const mlir::Attribute elementAttr : attr.getElts()) {
mlir::Attribute mlirAttr;
if (auto intAttr = mlir::dyn_cast<cir::IntAttr>(elementAttr)) {
mlirAttr = rewriter.getIntegerAttr(
converter->convertType(intAttr.getType()), intAttr.getValue());
} else if (auto floatAttr = mlir::dyn_cast<cir::FPAttr>(elementAttr)) {
mlirAttr = rewriter.getFloatAttr(
converter->convertType(floatAttr.getType()), floatAttr.getValue());
} else {
llvm_unreachable(
"vector constant with an element that is neither an int nor a float");
}
mlirValues.push_back(mlirAttr);
}
return mlir::LLVM::ConstantOp::create(
rewriter, loc, llvmTy,
mlir::DenseElementsAttr::get(mlir::cast<mlir::ShapedType>(llvmTy),
mlirValues));
}
// GlobalViewAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::GlobalViewAttr globalAttr) {
auto moduleOp = parentOp->getParentOfType<mlir::ModuleOp>();
mlir::DataLayout dataLayout(moduleOp);
mlir::Type sourceType;
assert(!cir::MissingFeatures::addressSpace());
llvm::StringRef symName;
mlir::Operation *sourceSymbol =
mlir::SymbolTable::lookupSymbolIn(moduleOp, globalAttr.getSymbol());
if (auto llvmSymbol = dyn_cast<mlir::LLVM::GlobalOp>(sourceSymbol)) {
sourceType = llvmSymbol.getType();
symName = llvmSymbol.getSymName();
} else if (auto cirSymbol = dyn_cast<cir::GlobalOp>(sourceSymbol)) {
sourceType =
convertTypeForMemory(*converter, dataLayout, cirSymbol.getSymType());
symName = cirSymbol.getSymName();
} else if (auto llvmFun = dyn_cast<mlir::LLVM::LLVMFuncOp>(sourceSymbol)) {
sourceType = llvmFun.getFunctionType();
symName = llvmFun.getSymName();
} else if (auto fun = dyn_cast<cir::FuncOp>(sourceSymbol)) {
sourceType = converter->convertType(fun.getFunctionType());
symName = fun.getSymName();
} else if (auto alias = dyn_cast<mlir::LLVM::AliasOp>(sourceSymbol)) {
sourceType = alias.getType();
symName = alias.getSymName();
} else {
llvm_unreachable("Unexpected GlobalOp type");
}
mlir::Location loc = parentOp->getLoc();
mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),
symName);
if (globalAttr.getIndices()) {
llvm::SmallVector<mlir::LLVM::GEPArg> indices;
if (mlir::isa<mlir::LLVM::LLVMArrayType, mlir::LLVM::LLVMStructType>(
sourceType))
indices.push_back(0);
for (mlir::Attribute idx : globalAttr.getIndices()) {
auto intAttr = mlir::cast<mlir::IntegerAttr>(idx);
indices.push_back(intAttr.getValue().getSExtValue());
}
mlir::Type resTy = addrOp.getType();
mlir::Type eltTy = converter->convertType(sourceType);
addrOp =
mlir::LLVM::GEPOp::create(rewriter, loc, resTy, eltTy, addrOp, indices,
mlir::LLVM::GEPNoWrapFlags::none);
}
// We can have a global view with an integer type in the case of method
// pointers. With the Itanium ABI, the #cir.method attribute is lowered to a
// #cir.global_view with a pointer-sized integer representing the address of
// the method.
if (auto intTy = mlir::dyn_cast<cir::IntType>(globalAttr.getType())) {
mlir::Type llvmDstTy = converter->convertType(globalAttr.getType());
return mlir::LLVM::PtrToIntOp::create(rewriter, parentOp->getLoc(),
llvmDstTy, addrOp);
}
if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(globalAttr.getType())) {
mlir::Type llvmEltTy =
convertTypeForMemory(*converter, dataLayout, ptrTy.getPointee());
if (llvmEltTy == sourceType)
return addrOp;
mlir::Type llvmDstTy = converter->convertType(globalAttr.getType());
return mlir::LLVM::BitcastOp::create(rewriter, parentOp->getLoc(),
llvmDstTy, addrOp);
}
llvm_unreachable("Expecting pointer or integer type for GlobalViewAttr");
}
// TypeInfoAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::TypeInfoAttr typeInfoAttr) {
mlir::Type llvmTy = converter->convertType(typeInfoAttr.getType());
mlir::Location loc = parentOp->getLoc();
mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
for (auto [idx, elt] : llvm::enumerate(typeInfoAttr.getData())) {
mlir::Value init = visit(elt);
result =
mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
}
return result;
}
/// UndefAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::UndefAttr undefAttr) {
mlir::Location loc = parentOp->getLoc();
return mlir::LLVM::UndefOp::create(
rewriter, loc, converter->convertType(undefAttr.getType()));
}
/// PoisonAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::PoisonAttr poisonAttr) {
mlir::Location loc = parentOp->getLoc();
return mlir::LLVM::PoisonOp::create(
rewriter, loc, converter->convertType(poisonAttr.getType()));
}
// VTableAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::VTableAttr vtableArr) {
mlir::Type llvmTy = converter->convertType(vtableArr.getType());
mlir::Location loc = parentOp->getLoc();
mlir::Value result = mlir::LLVM::UndefOp::create(rewriter, loc, llvmTy);
for (auto [idx, elt] : llvm::enumerate(vtableArr.getData())) {
mlir::Value init = visit(elt);
result =
mlir::LLVM::InsertValueOp::create(rewriter, loc, result, init, idx);
}
return result;
}
/// ZeroAttr visitor.
mlir::Value CIRAttrToValue::visitCirAttr(cir::ZeroAttr attr) {
mlir::Location loc = parentOp->getLoc();
return mlir::LLVM::ZeroOp::create(rewriter, loc,
converter->convertType(attr.getType()));
}
// This class handles rewriting initializer attributes for types that do not
// require region initialization.
class GlobalInitAttrRewriter {
public:
GlobalInitAttrRewriter(mlir::Type type,
mlir::ConversionPatternRewriter &rewriter)
: llvmType(type), rewriter(rewriter) {}
mlir::Attribute visit(mlir::Attribute attr) {
return llvm::TypeSwitch<mlir::Attribute, mlir::Attribute>(attr)
.Case<cir::IntAttr, cir::FPAttr, cir::BoolAttr>(
[&](auto attrT) { return visitCirAttr(attrT); })
.Default([&](auto attrT) { return mlir::Attribute(); });
}
mlir::Attribute visitCirAttr(cir::IntAttr attr) {
return rewriter.getIntegerAttr(llvmType, attr.getValue());
}
mlir::Attribute visitCirAttr(cir::FPAttr attr) {
return rewriter.getFloatAttr(llvmType, attr.getValue());
}
mlir::Attribute visitCirAttr(cir::BoolAttr attr) {
return rewriter.getBoolAttr(attr.getValue());
}
private:
mlir::Type llvmType;
mlir::ConversionPatternRewriter &rewriter;
};
// This pass requires the CIR to be in a "flat" state. All blocks in each
// function must belong to the parent region. Once scopes and control flow
// are implemented in CIR, a pass will be run before this one to flatten
// the CIR and get it into the state that this pass requires.
struct ConvertCIRToLLVMPass
: public mlir::PassWrapper<ConvertCIRToLLVMPass,
mlir::OperationPass<mlir::ModuleOp>> {
void getDependentDialects(mlir::DialectRegistry &registry) const override {
registry.insert<mlir::BuiltinDialect, mlir::DLTIDialect,
mlir::LLVM::LLVMDialect, mlir::func::FuncDialect>();
}
void runOnOperation() final;
void processCIRAttrs(mlir::ModuleOp module);
void resolveBlockAddressOp(LLVMBlockAddressInfo &blockInfoAddr);
StringRef getDescription() const override {
return "Convert the prepared CIR dialect module to LLVM dialect";
}
StringRef getArgument() const override { return "cir-flat-to-llvm"; }
};
mlir::LogicalResult CIRToLLVMIsFPClassOpLowering::matchAndRewrite(
cir::IsFPClassOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Value src = adaptor.getSrc();
cir::FPClassTest flags = adaptor.getFlags();
mlir::IntegerType retTy = rewriter.getI1Type();
rewriter.replaceOpWithNewOp<mlir::LLVM::IsFPClass>(
op, retTy, src, static_cast<uint32_t>(flags));
return mlir::success();
}
mlir::LogicalResult CIRToLLVMSignBitOpLowering::matchAndRewrite(
cir::SignBitOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
assert(!cir::MissingFeatures::isPPC_FP128Ty());
mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
int width = layout.getTypeSizeInBits(op.getInput().getType());
if (auto longDoubleType =
mlir::dyn_cast<cir::LongDoubleType>(op.getInput().getType())) {
if (mlir::isa<cir::FP80Type>(longDoubleType.getUnderlying())) {
// If the underlying type of LongDouble is FP80Type,
// DataLayout::getTypeSizeInBits returns 128.
// See https://github.com/llvm/clangir/issues/1057.
// Set the width to 80 manually.
width = 80;
}
}
mlir::Type intTy = mlir::IntegerType::get(rewriter.getContext(), width);
auto bitcast = mlir::LLVM::BitcastOp::create(rewriter, op->getLoc(), intTy,
adaptor.getInput());
auto zero = mlir::LLVM::ConstantOp::create(rewriter, op->getLoc(), intTy, 0);
auto cmpResult = mlir::LLVM::ICmpOp::create(rewriter, op.getLoc(),
mlir::LLVM::ICmpPredicate::slt,
bitcast.getResult(), zero);
rewriter.replaceOp(op, cmpResult);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAssumeOpLowering::matchAndRewrite(
cir::AssumeOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto cond = adaptor.getPredicate();
rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(op, cond);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAssumeAlignedOpLowering::matchAndRewrite(
cir::AssumeAlignedOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
SmallVector<mlir::Value, 3> opBundleArgs{adaptor.getPointer()};
auto alignment = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
adaptor.getAlignmentAttr());
opBundleArgs.push_back(alignment);
if (mlir::Value offset = adaptor.getOffset())
opBundleArgs.push_back(offset);
auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
rewriter.getI1Type(), 1);
mlir::LLVM::AssumeOp::create(rewriter, op.getLoc(), cond, "align",
opBundleArgs);
// The llvm.assume operation does not have a result, so we need to replace
// all uses of this cir.assume_aligned operation with the input ptr itself.
rewriter.replaceOp(op, adaptor.getPointer());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAssumeSepStorageOpLowering::matchAndRewrite(
cir::AssumeSepStorageOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto cond = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
rewriter.getI1Type(), 1);
rewriter.replaceOpWithNewOp<mlir::LLVM::AssumeOp>(
op, cond, mlir::LLVM::AssumeSeparateStorageTag{}, adaptor.getPtr1(),
adaptor.getPtr2());
return mlir::success();
}
static mlir::LLVM::AtomicOrdering
getLLVMMemOrder(std::optional<cir::MemOrder> memorder) {
if (!memorder)
return mlir::LLVM::AtomicOrdering::not_atomic;
switch (*memorder) {
case cir::MemOrder::Relaxed:
return mlir::LLVM::AtomicOrdering::monotonic;
case cir::MemOrder::Consume:
case cir::MemOrder::Acquire:
return mlir::LLVM::AtomicOrdering::acquire;
case cir::MemOrder::Release:
return mlir::LLVM::AtomicOrdering::release;
case cir::MemOrder::AcquireRelease:
return mlir::LLVM::AtomicOrdering::acq_rel;
case cir::MemOrder::SequentiallyConsistent:
return mlir::LLVM::AtomicOrdering::seq_cst;
}
llvm_unreachable("unknown memory order");
}
static llvm::StringRef getLLVMSyncScope(cir::SyncScopeKind syncScope) {
return syncScope == cir::SyncScopeKind::SingleThread ? "singlethread" : "";
}
static std::optional<llvm::StringRef>
getLLVMSyncScope(std::optional<cir::SyncScopeKind> syncScope) {
if (syncScope.has_value())
return getLLVMSyncScope(*syncScope);
return std::nullopt;
}
mlir::LogicalResult CIRToLLVMAtomicCmpXchgOpLowering::matchAndRewrite(
cir::AtomicCmpXchgOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Value expected = adaptor.getExpected();
mlir::Value desired = adaptor.getDesired();
auto cmpxchg = mlir::LLVM::AtomicCmpXchgOp::create(
rewriter, op.getLoc(), adaptor.getPtr(), expected, desired,
getLLVMMemOrder(adaptor.getSuccOrder()),
getLLVMMemOrder(adaptor.getFailOrder()),
getLLVMSyncScope(op.getSyncScope()));
cmpxchg.setAlignment(adaptor.getAlignment());
cmpxchg.setWeak(adaptor.getWeak());
cmpxchg.setVolatile_(adaptor.getIsVolatile());
// Check result and apply stores accordingly.
auto old = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(),
cmpxchg.getResult(), 0);
auto cmp = mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(),
cmpxchg.getResult(), 1);
rewriter.replaceOp(op, {old, cmp});
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAtomicXchgOpLowering::matchAndRewrite(
cir::AtomicXchgOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
assert(!cir::MissingFeatures::atomicSyncScopeID());
mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getMemOrder());
llvm::StringRef llvmSyncScope = getLLVMSyncScope(adaptor.getSyncScope());
rewriter.replaceOpWithNewOp<mlir::LLVM::AtomicRMWOp>(
op, mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(), adaptor.getVal(),
llvmOrder, llvmSyncScope);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAtomicTestAndSetOpLowering::matchAndRewrite(
cir::AtomicTestAndSetOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
assert(!cir::MissingFeatures::atomicSyncScopeID());
mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());
auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
rewriter.getI8Type(), 1);
auto rmw = mlir::LLVM::AtomicRMWOp::create(
rewriter, op.getLoc(), mlir::LLVM::AtomicBinOp::xchg, adaptor.getPtr(),
one, llvmOrder, /*syncscope=*/llvm::StringRef(),
adaptor.getAlignment().value_or(0), op.getIsVolatile());
auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
rewriter.getI8Type(), 0);
auto cmp = mlir::LLVM::ICmpOp::create(
rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, rmw, zero);
rewriter.replaceOp(op, cmp);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAtomicClearOpLowering::matchAndRewrite(
cir::AtomicClearOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
assert(!cir::MissingFeatures::atomicSyncScopeID());
mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());
auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
rewriter.getI8Type(), 0);
auto store = mlir::LLVM::StoreOp::create(
rewriter, op.getLoc(), zero, adaptor.getPtr(),
adaptor.getAlignment().value_or(0), op.getIsVolatile(),
/*isNonTemporal=*/false, /*isInvariantGroup=*/false, llvmOrder);
rewriter.replaceOp(op, store);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAtomicFenceOpLowering::matchAndRewrite(
cir::AtomicFenceOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(adaptor.getOrdering());
auto fence = mlir::LLVM::FenceOp::create(rewriter, op.getLoc(), llvmOrder);
fence.setSyncscope(getLLVMSyncScope(adaptor.getSyncscope()));
rewriter.replaceOp(op, fence);
return mlir::success();
}
static mlir::LLVM::AtomicBinOp
getLLVMAtomicBinOp(cir::AtomicFetchKind k, bool isInt, bool isSignedInt) {
switch (k) {
case cir::AtomicFetchKind::Add:
return isInt ? mlir::LLVM::AtomicBinOp::add : mlir::LLVM::AtomicBinOp::fadd;
case cir::AtomicFetchKind::Sub:
return isInt ? mlir::LLVM::AtomicBinOp::sub : mlir::LLVM::AtomicBinOp::fsub;
case cir::AtomicFetchKind::And:
return mlir::LLVM::AtomicBinOp::_and;
case cir::AtomicFetchKind::Xor:
return mlir::LLVM::AtomicBinOp::_xor;
case cir::AtomicFetchKind::Or:
return mlir::LLVM::AtomicBinOp::_or;
case cir::AtomicFetchKind::Nand:
return mlir::LLVM::AtomicBinOp::nand;
case cir::AtomicFetchKind::Max: {
if (!isInt)
return mlir::LLVM::AtomicBinOp::fmax;
return isSignedInt ? mlir::LLVM::AtomicBinOp::max
: mlir::LLVM::AtomicBinOp::umax;
}
case cir::AtomicFetchKind::Min: {
if (!isInt)
return mlir::LLVM::AtomicBinOp::fmin;
return isSignedInt ? mlir::LLVM::AtomicBinOp::min
: mlir::LLVM::AtomicBinOp::umin;
}
case cir::AtomicFetchKind::UIncWrap:
return mlir::LLVM::AtomicBinOp::uinc_wrap;
case cir::AtomicFetchKind::UDecWrap:
return mlir::LLVM::AtomicBinOp::udec_wrap;
}
llvm_unreachable("Unknown atomic fetch opcode");
}
static llvm::StringLiteral getLLVMBinopForPostAtomic(cir::AtomicFetchKind k,
bool isInt) {
switch (k) {
case cir::AtomicFetchKind::Add:
return isInt ? mlir::LLVM::AddOp::getOperationName()
: mlir::LLVM::FAddOp::getOperationName();
case cir::AtomicFetchKind::Sub:
return isInt ? mlir::LLVM::SubOp::getOperationName()
: mlir::LLVM::FSubOp::getOperationName();
case cir::AtomicFetchKind::And:
return mlir::LLVM::AndOp::getOperationName();
case cir::AtomicFetchKind::Xor:
return mlir::LLVM::XOrOp::getOperationName();
case cir::AtomicFetchKind::Or:
return mlir::LLVM::OrOp::getOperationName();
case cir::AtomicFetchKind::Nand:
// There's no nand binop in LLVM, this is later fixed with a not.
return mlir::LLVM::AndOp::getOperationName();
case cir::AtomicFetchKind::Max:
case cir::AtomicFetchKind::Min:
llvm_unreachable("handled in buildMinMaxPostOp");
case cir::AtomicFetchKind::UIncWrap:
case cir::AtomicFetchKind::UDecWrap:
llvm_unreachable("uinc_wrap and udec_wrap are always fetch_first");
}
llvm_unreachable("Unknown atomic fetch opcode");
}
mlir::Value CIRToLLVMAtomicFetchOpLowering::buildPostOp(
cir::AtomicFetchOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal,
bool isInt) const {
SmallVector<mlir::Value> atomicOperands = {rmwVal, adaptor.getVal()};
SmallVector<mlir::Type> atomicResTys = {rmwVal.getType()};
return rewriter
.create(op.getLoc(),
rewriter.getStringAttr(
getLLVMBinopForPostAtomic(op.getBinop(), isInt)),
atomicOperands, atomicResTys, {})
->getResult(0);
}
mlir::Value CIRToLLVMAtomicFetchOpLowering::buildMinMaxPostOp(
cir::AtomicFetchOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter, mlir::Value rmwVal, bool isInt,
bool isSigned) const {
mlir::Location loc = op.getLoc();
if (!isInt) {
if (op.getBinop() == cir::AtomicFetchKind::Max)
return mlir::LLVM::MaxNumOp::create(rewriter, loc, rmwVal,
adaptor.getVal());
return mlir::LLVM::MinNumOp::create(rewriter, loc, rmwVal,
adaptor.getVal());
}
mlir::LLVM::ICmpPredicate pred;
if (op.getBinop() == cir::AtomicFetchKind::Max) {
pred = isSigned ? mlir::LLVM::ICmpPredicate::sgt
: mlir::LLVM::ICmpPredicate::ugt;
} else { // Min
pred = isSigned ? mlir::LLVM::ICmpPredicate::slt
: mlir::LLVM::ICmpPredicate::ult;
}
mlir::Value cmp = mlir::LLVM::ICmpOp::create(
rewriter, loc,
mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(), pred), rmwVal,
adaptor.getVal());
return mlir::LLVM::SelectOp::create(rewriter, loc, cmp, rmwVal,
adaptor.getVal());
}
mlir::LogicalResult CIRToLLVMAtomicFetchOpLowering::matchAndRewrite(
cir::AtomicFetchOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
bool isInt = false;
bool isSignedInt = false;
if (auto intTy = mlir::dyn_cast<cir::IntType>(op.getVal().getType())) {
isInt = true;
isSignedInt = intTy.isSigned();
} else if (mlir::isa<cir::SingleType, cir::DoubleType>(
op.getVal().getType())) {
isInt = false;
} else {
return op.emitError() << "Unsupported type: " << op.getVal().getType();
}
mlir::LLVM::AtomicOrdering llvmOrder = getLLVMMemOrder(op.getMemOrder());
llvm::StringRef llvmSyncScope = getLLVMSyncScope(op.getSyncScope());
mlir::LLVM::AtomicBinOp llvmBinOp =
getLLVMAtomicBinOp(op.getBinop(), isInt, isSignedInt);
auto rmwVal = mlir::LLVM::AtomicRMWOp::create(
rewriter, op.getLoc(), llvmBinOp, adaptor.getPtr(), adaptor.getVal(),
llvmOrder, llvmSyncScope);
mlir::Value result = rmwVal.getResult();
if (!op.getFetchFirst()) {
if (op.getBinop() == cir::AtomicFetchKind::Max ||
op.getBinop() == cir::AtomicFetchKind::Min)
result = buildMinMaxPostOp(op, adaptor, rewriter, rmwVal.getRes(), isInt,
isSignedInt);
else
result = buildPostOp(op, adaptor, rewriter, rmwVal.getRes(), isInt);
// Compensate lack of nand binop in LLVM IR.
if (op.getBinop() == cir::AtomicFetchKind::Nand) {
auto negOne = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
result.getType(), -1);
result = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(), result, negOne);
}
}
rewriter.replaceOp(op, result);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMBitClrsbOpLowering::matchAndRewrite(
cir::BitClrsbOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
adaptor.getInput().getType(), 0);
auto isNeg = mlir::LLVM::ICmpOp::create(
rewriter, op.getLoc(),
mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
mlir::LLVM::ICmpPredicate::slt),
adaptor.getInput(), zero);
auto negOne = mlir::LLVM::ConstantOp::create(
rewriter, op.getLoc(), adaptor.getInput().getType(), -1);
auto flipped = mlir::LLVM::XOrOp::create(rewriter, op.getLoc(),
adaptor.getInput(), negOne);
auto select = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isNeg,
flipped, adaptor.getInput());
auto resTy = getTypeConverter()->convertType(op.getType());
auto clz = mlir::LLVM::CountLeadingZerosOp::create(
rewriter, op.getLoc(), resTy, select, /*is_zero_poison=*/false);
auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);
auto res = mlir::LLVM::SubOp::create(rewriter, op.getLoc(), clz, one,
mlir::LLVM::IntegerOverflowFlags::nuw);
rewriter.replaceOp(op, res);
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMBitClzOpLowering::matchAndRewrite(
cir::BitClzOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto resTy = getTypeConverter()->convertType(op.getType());
auto llvmOp = mlir::LLVM::CountLeadingZerosOp::create(
rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
rewriter.replaceOp(op, llvmOp);
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMBitCtzOpLowering::matchAndRewrite(
cir::BitCtzOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto resTy = getTypeConverter()->convertType(op.getType());
auto llvmOp = mlir::LLVM::CountTrailingZerosOp::create(
rewriter, op.getLoc(), resTy, adaptor.getInput(), op.getPoisonZero());
rewriter.replaceOp(op, llvmOp);
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMBitFfsOpLowering::matchAndRewrite(
cir::BitFfsOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto resTy = getTypeConverter()->convertType(op.getType());
auto ctz = mlir::LLVM::CountTrailingZerosOp::create(rewriter, op.getLoc(),
resTy, adaptor.getInput(),
/*is_zero_poison=*/true);
auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);
auto ctzAddOne = mlir::LLVM::AddOp::create(rewriter, op.getLoc(), ctz, one);
auto zeroInputTy = mlir::LLVM::ConstantOp::create(
rewriter, op.getLoc(), adaptor.getInput().getType(), 0);
auto isZero = mlir::LLVM::ICmpOp::create(
rewriter, op.getLoc(),
mlir::LLVM::ICmpPredicateAttr::get(rewriter.getContext(),
mlir::LLVM::ICmpPredicate::eq),
adaptor.getInput(), zeroInputTy);
auto zero = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 0);
auto res = mlir::LLVM::SelectOp::create(rewriter, op.getLoc(), isZero, zero,
ctzAddOne);
rewriter.replaceOp(op, res);
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMBitParityOpLowering::matchAndRewrite(
cir::BitParityOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto resTy = getTypeConverter()->convertType(op.getType());
auto popcnt = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy,
adaptor.getInput());
auto one = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(), resTy, 1);
auto popcntMod2 =
mlir::LLVM::AndOp::create(rewriter, op.getLoc(), popcnt, one);
rewriter.replaceOp(op, popcntMod2);
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMBitPopcountOpLowering::matchAndRewrite(
cir::BitPopcountOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto resTy = getTypeConverter()->convertType(op.getType());
auto llvmOp = mlir::LLVM::CtPopOp::create(rewriter, op.getLoc(), resTy,
adaptor.getInput());
rewriter.replaceOp(op, llvmOp);
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMBitReverseOpLowering::matchAndRewrite(
cir::BitReverseOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::BitReverseOp>(op, adaptor.getInput());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMBrCondOpLowering::matchAndRewrite(
cir::BrCondOp brOp, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// When ZExtOp is implemented, we'll need to check if the condition is a
// ZExtOp and if so, delete it if it has a single use.
assert(!cir::MissingFeatures::zextOp());
mlir::Value i1Condition = adaptor.getCond();
rewriter.replaceOpWithNewOp<mlir::LLVM::CondBrOp>(
brOp, i1Condition, brOp.getDestTrue(), adaptor.getDestOperandsTrue(),
brOp.getDestFalse(), adaptor.getDestOperandsFalse());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMByteSwapOpLowering::matchAndRewrite(
cir::ByteSwapOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::ByteSwapOp>(op, adaptor.getInput());
return mlir::LogicalResult::success();
}
mlir::Type CIRToLLVMCastOpLowering::convertTy(mlir::Type ty) const {
return getTypeConverter()->convertType(ty);
}
mlir::LogicalResult CIRToLLVMCastOpLowering::matchAndRewrite(
cir::CastOp castOp, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// For arithmetic conversions, LLVM IR uses the same instruction to convert
// both individual scalars and entire vectors. This lowering pass handles
// both situations.
switch (castOp.getKind()) {
case cir::CastKind::array_to_ptrdecay: {
const auto ptrTy = mlir::cast<cir::PointerType>(castOp.getType());
mlir::Value sourceValue = adaptor.getSrc();
mlir::Type targetType = convertTy(ptrTy);
mlir::Type elementTy = convertTypeForMemory(*getTypeConverter(), dataLayout,
ptrTy.getPointee());
llvm::SmallVector<mlir::LLVM::GEPArg> offset{0};
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
castOp, targetType, elementTy, sourceValue, offset);
break;
}
case cir::CastKind::int_to_bool: {
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Value zeroInt = mlir::LLVM::ConstantOp::create(
rewriter, castOp.getLoc(), llvmSrcVal.getType(), 0);
rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroInt);
break;
}
case cir::CastKind::integral: {
mlir::Type srcType = castOp.getSrc().getType();
mlir::Type dstType = castOp.getType();
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Type llvmDstType = getTypeConverter()->convertType(dstType);
cir::IntType srcIntType =
mlir::cast<cir::IntType>(elementTypeIfVector(srcType));
cir::IntType dstIntType =
mlir::cast<cir::IntType>(elementTypeIfVector(dstType));
rewriter.replaceOp(castOp, getLLVMIntCast(rewriter, llvmSrcVal, llvmDstType,
srcIntType.isUnsigned(),
srcIntType.getWidth(),
dstIntType.getWidth()));
break;
}
case cir::CastKind::floating: {
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Type llvmDstTy = getTypeConverter()->convertType(castOp.getType());
mlir::Type srcTy = elementTypeIfVector(castOp.getSrc().getType());
mlir::Type dstTy = elementTypeIfVector(castOp.getType());
if (!mlir::isa<cir::FPTypeInterface>(dstTy) ||
!mlir::isa<cir::FPTypeInterface>(srcTy))
return castOp.emitError() << "NYI cast from " << srcTy << " to " << dstTy;
auto getFloatWidth = [](mlir::Type ty) -> unsigned {
return mlir::cast<cir::FPTypeInterface>(ty).getWidth();
};
if (getFloatWidth(srcTy) > getFloatWidth(dstTy))
rewriter.replaceOpWithNewOp<mlir::LLVM::FPTruncOp>(castOp, llvmDstTy,
llvmSrcVal);
else
rewriter.replaceOpWithNewOp<mlir::LLVM::FPExtOp>(castOp, llvmDstTy,
llvmSrcVal);
return mlir::success();
}
case cir::CastKind::int_to_ptr: {
auto dstTy = mlir::cast<cir::PointerType>(castOp.getType());
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
rewriter.replaceOpWithNewOp<mlir::LLVM::IntToPtrOp>(castOp, llvmDstTy,
llvmSrcVal);
return mlir::success();
}
case cir::CastKind::ptr_to_int: {
auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
rewriter.replaceOpWithNewOp<mlir::LLVM::PtrToIntOp>(castOp, llvmDstTy,
llvmSrcVal);
return mlir::success();
}
case cir::CastKind::float_to_bool: {
mlir::Value llvmSrcVal = adaptor.getSrc();
auto kind = mlir::LLVM::FCmpPredicate::une;
// Check if float is not equal to zero.
auto zeroFloat = mlir::LLVM::ConstantOp::create(
rewriter, castOp.getLoc(), llvmSrcVal.getType(),
mlir::FloatAttr::get(llvmSrcVal.getType(), 0.0));
// Extend comparison result to either bool (C++) or int (C).
rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(castOp, kind, llvmSrcVal,
zeroFloat);
return mlir::success();
}
case cir::CastKind::bool_to_int: {
auto dstTy = mlir::cast<cir::IntType>(castOp.getType());
mlir::Value llvmSrcVal = adaptor.getSrc();
auto llvmSrcTy = mlir::cast<mlir::IntegerType>(llvmSrcVal.getType());
auto llvmDstTy =
mlir::cast<mlir::IntegerType>(getTypeConverter()->convertType(dstTy));
if (llvmSrcTy.getWidth() == llvmDstTy.getWidth())
rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
llvmSrcVal);
else
rewriter.replaceOpWithNewOp<mlir::LLVM::ZExtOp>(castOp, llvmDstTy,
llvmSrcVal);
return mlir::success();
}
case cir::CastKind::bool_to_float: {
mlir::Type dstTy = castOp.getType();
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
llvmSrcVal);
return mlir::success();
}
case cir::CastKind::int_to_float: {
mlir::Type dstTy = castOp.getType();
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getSrc().getType()))
.isSigned())
rewriter.replaceOpWithNewOp<mlir::LLVM::SIToFPOp>(castOp, llvmDstTy,
llvmSrcVal);
else
rewriter.replaceOpWithNewOp<mlir::LLVM::UIToFPOp>(castOp, llvmDstTy,
llvmSrcVal);
return mlir::success();
}
case cir::CastKind::float_to_int: {
mlir::Type dstTy = castOp.getType();
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
if (mlir::cast<cir::IntType>(elementTypeIfVector(castOp.getType()))
.isSigned())
rewriter.replaceOpWithNewOp<mlir::LLVM::FPToSIOp>(castOp, llvmDstTy,
llvmSrcVal);
else
rewriter.replaceOpWithNewOp<mlir::LLVM::FPToUIOp>(castOp, llvmDstTy,
llvmSrcVal);
return mlir::success();
}
case cir::CastKind::bitcast: {
mlir::Type dstTy = castOp.getType();
mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
assert(!MissingFeatures::cxxABI());
assert(!MissingFeatures::dataMemberType());
mlir::Value llvmSrcVal = adaptor.getSrc();
rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(castOp, llvmDstTy,
llvmSrcVal);
return mlir::success();
}
case cir::CastKind::ptr_to_bool: {
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Value zeroPtr = mlir::LLVM::ZeroOp::create(rewriter, castOp.getLoc(),
llvmSrcVal.getType());
rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
castOp, mlir::LLVM::ICmpPredicate::ne, llvmSrcVal, zeroPtr);
break;
}
case cir::CastKind::address_space: {
mlir::Type dstTy = castOp.getType();
mlir::Value llvmSrcVal = adaptor.getSrc();
mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
rewriter.replaceOpWithNewOp<mlir::LLVM::AddrSpaceCastOp>(castOp, llvmDstTy,
llvmSrcVal);
break;
}
case cir::CastKind::member_ptr_to_bool:
assert(!MissingFeatures::cxxABI());
assert(!MissingFeatures::methodType());
break;
default: {
return castOp.emitError("Unhandled cast kind: ")
<< castOp.getKindAttrName();
}
}
return mlir::success();
}
static mlir::Value convertToIndexTy(mlir::ConversionPatternRewriter &rewriter,
mlir::ModuleOp mod, mlir::Value index,
mlir::Type baseTy, cir::IntType strideTy) {
mlir::Operation *indexOp = index.getDefiningOp();
if (!indexOp)
return index;
auto indexType = mlir::cast<mlir::IntegerType>(index.getType());
mlir::DataLayout llvmLayout(mod);
std::optional<uint64_t> layoutWidth = llvmLayout.getTypeIndexBitwidth(baseTy);
// If there is no change in width, don't do anything.
if (!layoutWidth || *layoutWidth == indexType.getWidth())
return index;
// If the index comes from a subtraction, make sure the extension happens
// before it. To achieve that, look at unary minus, which already got
// lowered to "sub 0, x".
auto sub = dyn_cast<mlir::LLVM::SubOp>(indexOp);
bool rewriteSub = false;
if (sub) {
if (auto lhsConst =
dyn_cast<mlir::LLVM::ConstantOp>(sub.getLhs().getDefiningOp())) {
auto lhsConstInt = mlir::dyn_cast<mlir::IntegerAttr>(lhsConst.getValue());
if (lhsConstInt && lhsConstInt.getValue() == 0) {
index = sub.getRhs();
rewriteSub = true;
}
}
}
auto llvmDstType = rewriter.getIntegerType(*layoutWidth);
bool isUnsigned = strideTy && strideTy.isUnsigned();
index = getLLVMIntCast(rewriter, index, llvmDstType, isUnsigned,
indexType.getWidth(), *layoutWidth);
if (rewriteSub) {
index = mlir::LLVM::SubOp::create(
rewriter, index.getLoc(),
mlir::LLVM::ConstantOp::create(rewriter, index.getLoc(),
index.getType(), 0),
index);
// TODO: ensure sub is trivially dead now.
rewriter.eraseOp(sub);
}
return index;
}
mlir::LogicalResult CIRToLLVMPtrStrideOpLowering::matchAndRewrite(
cir::PtrStrideOp ptrStrideOp, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::TypeConverter *tc = getTypeConverter();
const mlir::Type resultTy = tc->convertType(ptrStrideOp.getType());
mlir::Type elementTy =
convertTypeForMemory(*tc, dataLayout, ptrStrideOp.getElementType());
// void and function types doesn't really have a layout to use in GEPs,
// make it i8 instead.
if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) ||
mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy))
elementTy = mlir::IntegerType::get(elementTy.getContext(), 8,
mlir::IntegerType::Signless);
// Zero-extend, sign-extend or trunc the pointer value.
mlir::Value index = adaptor.getStride();
index = convertToIndexTy(
rewriter, ptrStrideOp->getParentOfType<mlir::ModuleOp>(), index,
adaptor.getBase().getType(),
dyn_cast<cir::IntType>(ptrStrideOp.getOperand(1).getType()));
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
ptrStrideOp, resultTy, elementTy, adaptor.getBase(), index);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMGetElementOpLowering::matchAndRewrite(
cir::GetElementOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
if (auto arrayTy =
mlir::dyn_cast<cir::ArrayType>(op.getBaseType().getPointee())) {
const mlir::TypeConverter *converter = getTypeConverter();
const mlir::Type llArrayTy = converter->convertType(arrayTy);
const mlir::Type llResultTy = converter->convertType(op.getType());
mlir::Type elementTy =
convertTypeForMemory(*converter, dataLayout, op.getElementType());
// void and function types don't really have a layout to use in GEPs,
// make it i8 instead.
if (mlir::isa<mlir::LLVM::LLVMVoidType>(elementTy) ||
mlir::isa<mlir::LLVM::LLVMFunctionType>(elementTy))
elementTy = rewriter.getIntegerType(8);
mlir::Value index = adaptor.getIndex();
index =
convertToIndexTy(rewriter, op->getParentOfType<mlir::ModuleOp>(), index,
adaptor.getBase().getType(),
dyn_cast<cir::IntType>(op.getOperand(1).getType()));
// Since the base address is a pointer to an aggregate, the first
// offset is always zero. The second offset tell us which member it
// will access.
std::array<mlir::LLVM::GEPArg, 2> offset{0, index};
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, llResultTy, llArrayTy,
adaptor.getBase(), offset);
return mlir::success();
}
op.emitError() << "NYI: GetElementOp lowering to LLVM for non-array";
return mlir::failure();
}
mlir::LogicalResult CIRToLLVMBaseClassAddrOpLowering::matchAndRewrite(
cir::BaseClassAddrOp baseClassOp, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type resultType =
getTypeConverter()->convertType(baseClassOp.getType());
mlir::Value derivedAddr = adaptor.getDerivedAddr();
llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {
adaptor.getOffset().getZExtValue()};
mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,
mlir::IntegerType::Signless);
if (adaptor.getOffset().getZExtValue() == 0) {
rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(
baseClassOp, resultType, adaptor.getDerivedAddr());
return mlir::success();
}
if (baseClassOp.getAssumeNotNull()) {
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
baseClassOp, resultType, byteType, derivedAddr, offset);
} else {
auto loc = baseClassOp.getLoc();
mlir::Value isNull = mlir::LLVM::ICmpOp::create(
rewriter, loc, mlir::LLVM::ICmpPredicate::eq, derivedAddr,
mlir::LLVM::ZeroOp::create(rewriter, loc, derivedAddr.getType()));
mlir::Value adjusted = mlir::LLVM::GEPOp::create(
rewriter, loc, resultType, byteType, derivedAddr, offset);
rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(baseClassOp, isNull,
derivedAddr, adjusted);
}
return mlir::success();
}
mlir::LogicalResult CIRToLLVMDerivedClassAddrOpLowering::matchAndRewrite(
cir::DerivedClassAddrOp derivedClassOp, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type resultType =
getTypeConverter()->convertType(derivedClassOp.getType());
mlir::Value baseAddr = adaptor.getBaseAddr();
// The offset is set in the operation as an unsigned value, but it must be
// applied as a negative offset.
int64_t offsetVal = -(adaptor.getOffset().getZExtValue());
if (offsetVal == 0) {
// If the offset is zero, we can just return the base address,
rewriter.replaceOp(derivedClassOp, baseAddr);
return mlir::success();
}
llvm::SmallVector<mlir::LLVM::GEPArg, 1> offset = {offsetVal};
mlir::Type byteType = mlir::IntegerType::get(resultType.getContext(), 8,
mlir::IntegerType::Signless);
if (derivedClassOp.getAssumeNotNull()) {
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
derivedClassOp, resultType, byteType, baseAddr, offset,
mlir::LLVM::GEPNoWrapFlags::inbounds);
} else {
mlir::Location loc = derivedClassOp.getLoc();
mlir::Value isNull = mlir::LLVM::ICmpOp::create(
rewriter, loc, mlir::LLVM::ICmpPredicate::eq, baseAddr,
mlir::LLVM::ZeroOp::create(rewriter, loc, baseAddr.getType()));
mlir::Value adjusted =
mlir::LLVM::GEPOp::create(rewriter, loc, resultType, byteType, baseAddr,
offset, mlir::LLVM::GEPNoWrapFlags::inbounds);
rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(derivedClassOp, isNull,
baseAddr, adjusted);
}
return mlir::success();
}
mlir::LogicalResult CIRToLLVMFMaxNumOpLowering::matchAndRewrite(
cir::FMaxNumOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type resTy = typeConverter->convertType(op.getType());
rewriter.replaceOpWithNewOp<mlir::LLVM::MaxNumOp>(
op, resTy, adaptor.getLhs(), adaptor.getRhs(),
mlir::LLVM::FastmathFlags::nsz);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMFMinNumOpLowering::matchAndRewrite(
cir::FMinNumOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type resTy = typeConverter->convertType(op.getType());
rewriter.replaceOpWithNewOp<mlir::LLVM::MinNumOp>(
op, resTy, adaptor.getLhs(), adaptor.getRhs(),
mlir::LLVM::FastmathFlags::nsz);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAllocaOpLowering::matchAndRewrite(
cir::AllocaOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Value size =
op.isDynamic()
? adaptor.getDynAllocSize()
: mlir::LLVM::ConstantOp::create(
rewriter, op.getLoc(),
typeConverter->convertType(rewriter.getIndexType()), 1);
mlir::Type elementTy =
convertTypeForMemory(*getTypeConverter(), dataLayout, op.getAllocaType());
mlir::Type resultTy =
convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
assert(!cir::MissingFeatures::addressSpace());
assert(!cir::MissingFeatures::opAllocaAnnotations());
rewriter.replaceOpWithNewOp<mlir::LLVM::AllocaOp>(op, resultTy, elementTy,
size, op.getAlignment());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMReturnOpLowering::matchAndRewrite(
cir::ReturnOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::ReturnOp>(op, adaptor.getOperands());
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMRotateOpLowering::matchAndRewrite(
cir::RotateOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// Note that LLVM intrinsic calls to @llvm.fsh{r,l}.i* have the same type as
// the operand.
mlir::Value input = adaptor.getInput();
if (op.isRotateLeft())
rewriter.replaceOpWithNewOp<mlir::LLVM::FshlOp>(op, input, input,
adaptor.getAmount());
else
rewriter.replaceOpWithNewOp<mlir::LLVM::FshrOp>(op, input, input,
adaptor.getAmount());
return mlir::LogicalResult::success();
}
static void lowerCallAttributes(cir::CIRCallOpInterface op,
SmallVectorImpl<mlir::NamedAttribute> &result) {
for (mlir::NamedAttribute attr : op->getAttrs()) {
assert(!cir::MissingFeatures::opFuncCallingConv());
if (attr.getName() == CIRDialect::getCalleeAttrName() ||
attr.getName() == CIRDialect::getSideEffectAttrName() ||
attr.getName() == CIRDialect::getNoThrowAttrName() ||
attr.getName() == CIRDialect::getNoUnwindAttrName() ||
attr.getName() == CIRDialect::getNoReturnAttrName())
continue;
assert(!cir::MissingFeatures::opFuncExtraAttrs());
result.push_back(attr);
}
}
static mlir::LogicalResult
rewriteCallOrInvoke(mlir::Operation *op, mlir::ValueRange callOperands,
mlir::ConversionPatternRewriter &rewriter,
const mlir::TypeConverter *converter,
mlir::FlatSymbolRefAttr calleeAttr,
mlir::Block *continueBlock = nullptr,
mlir::Block *landingPadBlock = nullptr) {
llvm::SmallVector<mlir::Type, 8> llvmResults;
mlir::ValueTypeRange<mlir::ResultRange> cirResults = op->getResultTypes();
auto call = cast<cir::CIRCallOpInterface>(op);
if (converter->convertTypes(cirResults, llvmResults).failed())
return mlir::failure();
assert(!cir::MissingFeatures::opCallCallConv());
mlir::LLVM::MemoryEffectsAttr memoryEffects;
bool noUnwind = false;
bool willReturn = false;
bool noReturn = false;
convertSideEffectForCall(op, call.getNothrow(), call.getSideEffect(),
memoryEffects, noUnwind, willReturn, noReturn);
SmallVector<mlir::NamedAttribute, 4> attributes;
lowerCallAttributes(call, attributes);
mlir::LLVM::LLVMFunctionType llvmFnTy;
// Temporary to handle the case where we need to prepend an operand if the
// callee is an alias.
SmallVector<mlir::Value> adjustedCallOperands;
if (calleeAttr) { // direct call
mlir::Operation *callee =
mlir::SymbolTable::lookupNearestSymbolFrom(op, calleeAttr);
if (auto fn = mlir::dyn_cast<mlir::FunctionOpInterface>(callee)) {
llvmFnTy = converter->convertType<mlir::LLVM::LLVMFunctionType>(
fn.getFunctionType());
assert(llvmFnTy && "Failed to convert function type");
} else if (auto alias = mlir::cast<mlir::LLVM::AliasOp>(callee)) {
// If the callee was an alias. In that case,
// we need to prepend the address of the alias to the operands. The
// way aliases work in the LLVM dialect is a little counter-intuitive.
// The AliasOp itself is a pseudo-function that returns the address of
// the global value being aliased, but when we generate the call we
// need to insert an operation that gets the address of the AliasOp.
// This all gets sorted out when the LLVM dialect is lowered to LLVM IR.
auto symAttr = mlir::cast<mlir::FlatSymbolRefAttr>(calleeAttr);
auto addrOfAlias =
mlir::LLVM::AddressOfOp::create(
rewriter, op->getLoc(),
mlir::LLVM::LLVMPointerType::get(rewriter.getContext()), symAttr)
.getResult();
adjustedCallOperands.push_back(addrOfAlias);
// Now add the regular operands and assign this to the range value.
llvm::append_range(adjustedCallOperands, callOperands);
callOperands = adjustedCallOperands;
// Clear the callee attribute because we're calling an alias.
calleeAttr = {};
llvmFnTy = mlir::cast<mlir::LLVM::LLVMFunctionType>(alias.getType());
} else {
// Was this an ifunc?
return op->emitError("Unexpected callee type!");
}
} else { // indirect call
assert(!op->getOperands().empty() &&
"operands list must no be empty for the indirect call");
auto calleeTy = op->getOperands().front().getType();
auto calleePtrTy = cast<cir::PointerType>(calleeTy);
auto calleeFuncTy = cast<cir::FuncType>(calleePtrTy.getPointee());
llvm::append_range(adjustedCallOperands, callOperands);
llvmFnTy = cast<mlir::LLVM::LLVMFunctionType>(
converter->convertType(calleeFuncTy));
}
assert(!cir::MissingFeatures::opCallCallConv());
if (landingPadBlock) {
auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::InvokeOp>(
op, llvmFnTy, calleeAttr, callOperands, continueBlock,
mlir::ValueRange{}, landingPadBlock, mlir::ValueRange{});
newOp->setAttrs(attributes);
} else {
auto newOp = rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
op, llvmFnTy, calleeAttr, callOperands);
newOp->setAttrs(attributes);
if (memoryEffects)
newOp.setMemoryEffectsAttr(memoryEffects);
newOp.setNoUnwind(noUnwind);
newOp.setWillReturn(willReturn);
newOp.setNoreturn(noReturn);
}
return mlir::success();
}
mlir::LogicalResult CIRToLLVMCallOpLowering::matchAndRewrite(
cir::CallOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return rewriteCallOrInvoke(op.getOperation(), adaptor.getOperands(), rewriter,
getTypeConverter(), op.getCalleeAttr());
}
mlir::LogicalResult CIRToLLVMTryCallOpLowering::matchAndRewrite(
cir::TryCallOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
assert(!cir::MissingFeatures::opCallCallConv());
return rewriteCallOrInvoke(op.getOperation(), adaptor.getOperands(), rewriter,
getTypeConverter(), op.getCalleeAttr(),
op.getNormalDest(), op.getUnwindDest());
}
mlir::LogicalResult CIRToLLVMReturnAddrOpLowering::matchAndRewrite(
cir::ReturnAddrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType());
replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.returnaddress",
llvmPtrTy, adaptor.getOperands());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMFrameAddrOpLowering::matchAndRewrite(
cir::FrameAddrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType());
replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.frameaddress", llvmPtrTy,
adaptor.getOperands());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMClearCacheOpLowering::matchAndRewrite(
cir::ClearCacheOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Value begin = adaptor.getBegin();
mlir::Value end = adaptor.getEnd();
auto intrinNameAttr =
mlir::StringAttr::get(op.getContext(), "llvm.clear_cache");
rewriter.replaceOpWithNewOp<mlir::LLVM::CallIntrinsicOp>(
op, mlir::Type{}, intrinNameAttr, mlir::ValueRange{begin, end});
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAddrOfReturnAddrOpLowering::matchAndRewrite(
cir::AddrOfReturnAddrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type llvmPtrTy = getTypeConverter()->convertType(op.getType());
replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.addressofreturnaddress",
llvmPtrTy, adaptor.getOperands());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMLoadOpLowering::matchAndRewrite(
cir::LoadOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type llvmTy =
convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
mlir::LLVM::AtomicOrdering ordering = getLLVMMemOrder(op.getMemOrder());
std::optional<size_t> opAlign = op.getAlignment();
unsigned alignment =
(unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
assert(!cir::MissingFeatures::lowerModeOptLevel());
// TODO: nontemporal.
assert(!cir::MissingFeatures::opLoadStoreNontemporal());
std::optional<llvm::StringRef> llvmSyncScope =
getLLVMSyncScope(op.getSyncScope());
mlir::LLVM::LoadOp newLoad = mlir::LLVM::LoadOp::create(
rewriter, op->getLoc(), llvmTy, adaptor.getAddr(), alignment,
op.getIsVolatile(), /*isNonTemporal=*/false,
/*isInvariant=*/false, /*isInvariantGroup=*/false, ordering,
llvmSyncScope.value_or(std::string()));
// Convert adapted result to its original type if needed.
mlir::Value result =
emitFromMemory(rewriter, dataLayout, op, newLoad.getResult());
rewriter.replaceOp(op, result);
assert(!cir::MissingFeatures::opLoadStoreTbaa());
return mlir::LogicalResult::success();
}
mlir::LogicalResult
cir::direct::CIRToLLVMVecMaskedLoadOpLowering::matchAndRewrite(
cir::VecMaskedLoadOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type llvmResTy =
convertTypeForMemory(*getTypeConverter(), dataLayout, op.getType());
std::optional<size_t> opAlign = op.getAlignment();
unsigned alignment =
(unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmResTy));
mlir::IntegerAttr alignAttr = rewriter.getI32IntegerAttr(alignment);
auto newLoad = mlir::LLVM::MaskedLoadOp::create(
rewriter, op.getLoc(), llvmResTy, adaptor.getAddr(), adaptor.getMask(),
adaptor.getPassThru(), alignAttr);
rewriter.replaceOp(op, newLoad.getResult());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMStoreOpLowering::matchAndRewrite(
cir::StoreOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::LLVM::AtomicOrdering memorder = getLLVMMemOrder(op.getMemOrder());
const mlir::Type llvmTy =
getTypeConverter()->convertType(op.getValue().getType());
std::optional<size_t> opAlign = op.getAlignment();
unsigned alignment =
(unsigned)opAlign.value_or(dataLayout.getTypeABIAlignment(llvmTy));
assert(!cir::MissingFeatures::lowerModeOptLevel());
// Convert adapted value to its memory type if needed.
mlir::Value value = emitToMemory(rewriter, dataLayout,
op.getValue().getType(), adaptor.getValue());
// TODO: nontemporal.
assert(!cir::MissingFeatures::opLoadStoreNontemporal());
assert(!cir::MissingFeatures::opLoadStoreTbaa());
std::optional<llvm::StringRef> llvmSyncScope =
getLLVMSyncScope(op.getSyncScope());
mlir::LLVM::StoreOp storeOp = mlir::LLVM::StoreOp::create(
rewriter, op->getLoc(), value, adaptor.getAddr(), alignment,
op.getIsVolatile(),
/*isNonTemporal=*/false, /*isInvariantGroup=*/false, memorder,
llvmSyncScope.value_or(std::string()));
rewriter.replaceOp(op, storeOp);
assert(!cir::MissingFeatures::opLoadStoreTbaa());
return mlir::LogicalResult::success();
}
bool hasTrailingZeros(cir::ConstArrayAttr attr) {
auto array = mlir::dyn_cast<mlir::ArrayAttr>(attr.getElts());
return attr.hasTrailingZeros() ||
(array && std::count_if(array.begin(), array.end(), [](auto elt) {
auto ar = dyn_cast<cir::ConstArrayAttr>(elt);
return ar && hasTrailingZeros(ar);
}));
}
mlir::LogicalResult CIRToLLVMConstantOpLowering::matchAndRewrite(
cir::ConstantOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Attribute attr = op.getValue();
if (mlir::isa<cir::PoisonAttr>(attr)) {
rewriter.replaceOpWithNewOp<mlir::LLVM::PoisonOp>(
op, getTypeConverter()->convertType(op.getType()));
return mlir::success();
}
if (mlir::isa<mlir::IntegerType>(op.getType())) {
// Verified cir.const operations cannot actually be of these types, but the
// lowering pass may generate temporary cir.const operations with these
// types. This is OK since MLIR allows unverified operations to be alive
// during a pass as long as they don't live past the end of the pass.
attr = op.getValue();
} else if (mlir::isa<cir::BoolType>(op.getType())) {
int value = mlir::cast<cir::BoolAttr>(op.getValue()).getValue();
attr = rewriter.getIntegerAttr(typeConverter->convertType(op.getType()),
value);
} else if (mlir::isa<cir::IntType>(op.getType())) {
// Lower GlobalViewAttr to llvm.mlir.addressof + llvm.mlir.ptrtoint
if (auto ga = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
// We can have a global view with an integer type in the case of method
// pointers, but the lowering of those doesn't go through this path.
// They are handled in the visitCirAttr. This is left as an error until
// we have a test case that reaches it.
assert(!cir::MissingFeatures::globalViewIntLowering());
op.emitError() << "global view with integer type";
return mlir::failure();
}
attr = rewriter.getIntegerAttr(
typeConverter->convertType(op.getType()),
mlir::cast<cir::IntAttr>(op.getValue()).getValue());
} else if (mlir::isa<cir::FPTypeInterface>(op.getType())) {
attr = rewriter.getFloatAttr(
typeConverter->convertType(op.getType()),
mlir::cast<cir::FPAttr>(op.getValue()).getValue());
} else if (mlir::isa<cir::PointerType>(op.getType())) {
// Optimize with dedicated LLVM op for null pointers.
if (mlir::isa<cir::ConstPtrAttr>(op.getValue())) {
if (mlir::cast<cir::ConstPtrAttr>(op.getValue()).isNullValue()) {
rewriter.replaceOpWithNewOp<mlir::LLVM::ZeroOp>(
op, typeConverter->convertType(op.getType()));
return mlir::success();
}
}
// Lower GlobalViewAttr to llvm.mlir.addressof
if (auto gv = mlir::dyn_cast<cir::GlobalViewAttr>(op.getValue())) {
auto newOp = lowerCirAttrAsValue(op, gv, rewriter, getTypeConverter());
rewriter.replaceOp(op, newOp);
return mlir::success();
}
attr = op.getValue();
} else if (const auto arrTy = mlir::dyn_cast<cir::ArrayType>(op.getType())) {
const auto constArr = mlir::dyn_cast<cir::ConstArrayAttr>(op.getValue());
if (!constArr && !isa<cir::ZeroAttr, cir::UndefAttr>(op.getValue()))
return op.emitError() << "array does not have a constant initializer";
std::optional<mlir::Attribute> denseAttr;
if (constArr && hasTrailingZeros(constArr)) {
const mlir::Value newOp =
lowerCirAttrAsValue(op, constArr, rewriter, getTypeConverter());
rewriter.replaceOp(op, newOp);
return mlir::success();
} else if (constArr &&
(denseAttr = lowerConstArrayAttr(constArr, typeConverter))) {
attr = denseAttr.value();
} else {
const mlir::Value initVal =
lowerCirAttrAsValue(op, op.getValue(), rewriter, typeConverter);
rewriter.replaceOp(op, initVal);
return mlir::success();
}
} else if (const auto recordAttr =
mlir::dyn_cast<cir::ConstRecordAttr>(op.getValue())) {
auto initVal = lowerCirAttrAsValue(op, recordAttr, rewriter, typeConverter);
rewriter.replaceOp(op, initVal);
return mlir::success();
} else if (const auto vecTy = mlir::dyn_cast<cir::VectorType>(op.getType())) {
rewriter.replaceOp(op, lowerCirAttrAsValue(op, op.getValue(), rewriter,
getTypeConverter()));
return mlir::success();
} else if (auto recTy = mlir::dyn_cast<cir::RecordType>(op.getType())) {
if (mlir::isa<cir::ZeroAttr, cir::UndefAttr>(attr)) {
mlir::Value initVal =
lowerCirAttrAsValue(op, attr, rewriter, typeConverter);
rewriter.replaceOp(op, initVal);
return mlir::success();
}
return op.emitError() << "unsupported lowering for record constant type "
<< op.getType();
} else if (auto complexTy = mlir::dyn_cast<cir::ComplexType>(op.getType())) {
mlir::Type complexElemTy = complexTy.getElementType();
mlir::Type complexElemLLVMTy = typeConverter->convertType(complexElemTy);
if (auto zeroInitAttr = mlir::dyn_cast<cir::ZeroAttr>(op.getValue())) {
mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(complexElemLLVMTy);
mlir::ArrayAttr array = rewriter.getArrayAttr({zeroAttr, zeroAttr});
rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
op, getTypeConverter()->convertType(op.getType()), array);
return mlir::success();
}
auto complexAttr = mlir::cast<cir::ConstComplexAttr>(op.getValue());
mlir::Attribute components[2];
if (mlir::isa<cir::IntType>(complexElemTy)) {
components[0] = rewriter.getIntegerAttr(
complexElemLLVMTy,
mlir::cast<cir::IntAttr>(complexAttr.getReal()).getValue());
components[1] = rewriter.getIntegerAttr(
complexElemLLVMTy,
mlir::cast<cir::IntAttr>(complexAttr.getImag()).getValue());
} else {
components[0] = rewriter.getFloatAttr(
complexElemLLVMTy,
mlir::cast<cir::FPAttr>(complexAttr.getReal()).getValue());
components[1] = rewriter.getFloatAttr(
complexElemLLVMTy,
mlir::cast<cir::FPAttr>(complexAttr.getImag()).getValue());
}
attr = rewriter.getArrayAttr(components);
} else {
return op.emitError() << "unsupported constant type " << op.getType();
}
rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
op, getTypeConverter()->convertType(op.getType()), attr);
return mlir::success();
}
static uint64_t getTypeSize(mlir::Type type, mlir::Operation &op) {
mlir::DataLayout layout(op.getParentOfType<mlir::ModuleOp>());
// For LLVM purposes we treat void as u8.
if (isa<cir::VoidType>(type))
type = cir::IntType::get(type.getContext(), 8, /*isSigned=*/false);
return llvm::divideCeil(layout.getTypeSizeInBits(type), 8);
}
mlir::LogicalResult CIRToLLVMPrefetchOpLowering::matchAndRewrite(
cir::PrefetchOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::Prefetch>(
op, adaptor.getAddr(), adaptor.getIsWrite(), adaptor.getLocality(),
/*DataCache=*/1);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMPtrDiffOpLowering::matchAndRewrite(
cir::PtrDiffOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto dstTy = mlir::cast<cir::IntType>(op.getType());
mlir::Type llvmDstTy = getTypeConverter()->convertType(dstTy);
auto lhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy,
adaptor.getLhs());
auto rhs = mlir::LLVM::PtrToIntOp::create(rewriter, op.getLoc(), llvmDstTy,
adaptor.getRhs());
auto diff =
mlir::LLVM::SubOp::create(rewriter, op.getLoc(), llvmDstTy, lhs, rhs);
cir::PointerType ptrTy = op.getLhs().getType();
assert(!cir::MissingFeatures::llvmLoweringPtrDiffConsidersPointee());
uint64_t typeSize = getTypeSize(ptrTy.getPointee(), *op);
// Avoid silly division by 1.
mlir::Value resultVal = diff.getResult();
if (typeSize != 1) {
auto typeSizeVal = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
llvmDstTy, typeSize);
if (dstTy.isUnsigned()) {
auto uDiv =
mlir::LLVM::UDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal);
uDiv.setIsExact(true);
resultVal = uDiv.getResult();
} else {
auto sDiv =
mlir::LLVM::SDivOp::create(rewriter, op.getLoc(), diff, typeSizeVal);
sDiv.setIsExact(true);
resultVal = sDiv.getResult();
}
}
rewriter.replaceOp(op, resultVal);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMExpectOpLowering::matchAndRewrite(
cir::ExpectOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// TODO(cir): do not generate LLVM intrinsics under -O0
assert(!cir::MissingFeatures::optInfoAttr());
std::optional<llvm::APFloat> prob = op.getProb();
if (prob)
rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectWithProbabilityOp>(
op, adaptor.getVal(), adaptor.getExpected(), prob.value());
else
rewriter.replaceOpWithNewOp<mlir::LLVM::ExpectOp>(op, adaptor.getVal(),
adaptor.getExpected());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAbsOpLowering::matchAndRewrite(
cir::AbsOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type resTy = typeConverter->convertType(op.getType());
auto absOp = mlir::LLVM::AbsOp::create(rewriter, op.getLoc(), resTy,
adaptor.getOperands()[0],
adaptor.getMinIsPoison());
rewriter.replaceOp(op, absOp);
return mlir::success();
}
/// Convert the `cir.func` attributes to `llvm.func` attributes.
/// Only retain those attributes that are not constructed by
/// `LLVMFuncOp::build`. If `filterArgAttrs` is set, also filter out
/// argument attributes.
void CIRToLLVMFuncOpLowering::lowerFuncAttributes(
cir::FuncOp func, bool filterArgAndResAttrs,
SmallVectorImpl<mlir::NamedAttribute> &result) const {
assert(!cir::MissingFeatures::opFuncCallingConv());
for (mlir::NamedAttribute attr : func->getAttrs()) {
assert(!cir::MissingFeatures::opFuncCallingConv());
if (attr.getName() == mlir::SymbolTable::getSymbolAttrName() ||
attr.getName() == func.getFunctionTypeAttrName() ||
attr.getName() == getLinkageAttrNameString() ||
attr.getName() == func.getDsoLocalAttrName() ||
attr.getName() == func.getInlineKindAttrName() ||
attr.getName() == func.getSideEffectAttrName() ||
attr.getName() == CIRDialect::getNoReturnAttrName() ||
(filterArgAndResAttrs &&
(attr.getName() == func.getArgAttrsAttrName() ||
attr.getName() == func.getResAttrsAttrName())))
continue;
assert(!cir::MissingFeatures::opFuncExtraAttrs());
result.push_back(attr);
}
}
mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewriteAlias(
cir::FuncOp op, llvm::StringRef aliasee, mlir::Type ty, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
SmallVector<mlir::NamedAttribute, 4> attributes;
lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes);
mlir::Location loc = op.getLoc();
auto aliasOp = rewriter.replaceOpWithNewOp<mlir::LLVM::AliasOp>(
op, ty, convertLinkage(op.getLinkage()), op.getName(), op.getDsoLocal(),
/*threadLocal=*/false, attributes);
// Create the alias body
mlir::OpBuilder builder(op.getContext());
mlir::Block *block = builder.createBlock(&aliasOp.getInitializerRegion());
builder.setInsertionPointToStart(block);
// The type of AddressOfOp is always a pointer.
assert(!cir::MissingFeatures::addressSpace());
mlir::Type ptrTy = mlir::LLVM::LLVMPointerType::get(ty.getContext());
auto addrOp = mlir::LLVM::AddressOfOp::create(builder, loc, ptrTy, aliasee);
mlir::LLVM::ReturnOp::create(builder, loc, addrOp);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMFuncOpLowering::matchAndRewrite(
cir::FuncOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
cir::FuncType fnType = op.getFunctionType();
bool isDsoLocal = op.getDsoLocal();
mlir::TypeConverter::SignatureConversion signatureConversion(
fnType.getNumInputs());
for (const auto &argType : llvm::enumerate(fnType.getInputs())) {
mlir::Type convertedType = typeConverter->convertType(argType.value());
if (!convertedType)
return mlir::failure();
signatureConversion.addInputs(argType.index(), convertedType);
}
mlir::Type resultType =
getTypeConverter()->convertType(fnType.getReturnType());
// Create the LLVM function operation.
mlir::Type llvmFnTy = mlir::LLVM::LLVMFunctionType::get(
resultType ? resultType : mlir::LLVM::LLVMVoidType::get(getContext()),
signatureConversion.getConvertedTypes(),
/*isVarArg=*/fnType.isVarArg());
// If this is an alias, it needs to be lowered to llvm::AliasOp.
if (std::optional<llvm::StringRef> aliasee = op.getAliasee())
return matchAndRewriteAlias(op, *aliasee, llvmFnTy, adaptor, rewriter);
// LLVMFuncOp expects a single FileLine Location instead of a fused
// location.
mlir::Location loc = op.getLoc();
if (mlir::FusedLoc fusedLoc = mlir::dyn_cast<mlir::FusedLoc>(loc))
loc = fusedLoc.getLocations()[0];
assert((mlir::isa<mlir::FileLineColLoc>(loc) ||
mlir::isa<mlir::UnknownLoc>(loc)) &&
"expected single location or unknown location here");
mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
assert(!cir::MissingFeatures::opFuncCallingConv());
mlir::LLVM::CConv cconv = mlir::LLVM::CConv::C;
SmallVector<mlir::NamedAttribute, 4> attributes;
lowerFuncAttributes(op, /*filterArgAndResAttrs=*/false, attributes);
mlir::LLVM::LLVMFuncOp fn = mlir::LLVM::LLVMFuncOp::create(
rewriter, loc, op.getName(), llvmFnTy, linkage, isDsoLocal, cconv,
mlir::SymbolRefAttr(), attributes);
assert(!cir::MissingFeatures::opFuncMultipleReturnVals());
if (std::optional<cir::SideEffect> sideEffectKind = op.getSideEffect()) {
switch (*sideEffectKind) {
case cir::SideEffect::All:
break;
case cir::SideEffect::Pure:
fn.setMemoryEffectsAttr(mlir::LLVM::MemoryEffectsAttr::get(
fn.getContext(),
/*other=*/mlir::LLVM::ModRefInfo::Ref,
/*argMem=*/mlir::LLVM::ModRefInfo::Ref,
/*inaccessibleMem=*/mlir::LLVM::ModRefInfo::Ref,
/*errnoMem=*/mlir::LLVM::ModRefInfo::Ref,
/*targetMem0=*/mlir::LLVM::ModRefInfo::Ref,
/*targetMem1=*/mlir::LLVM::ModRefInfo::Ref));
fn.setNoUnwind(true);
fn.setWillReturn(true);
break;
case cir::SideEffect::Const:
fn.setMemoryEffectsAttr(mlir::LLVM::MemoryEffectsAttr::get(
fn.getContext(),
/*other=*/mlir::LLVM::ModRefInfo::NoModRef,
/*argMem=*/mlir::LLVM::ModRefInfo::NoModRef,
/*inaccessibleMem=*/mlir::LLVM::ModRefInfo::NoModRef,
/*errnoMem=*/mlir::LLVM::ModRefInfo::NoModRef,
/*targetMem0=*/mlir::LLVM::ModRefInfo::NoModRef,
/*targetMem1=*/mlir::LLVM::ModRefInfo::NoModRef));
fn.setNoUnwind(true);
fn.setWillReturn(true);
break;
}
}
if (op->hasAttr(CIRDialect::getNoReturnAttrName()))
fn.setNoreturn(true);
if (std::optional<cir::InlineKind> inlineKind = op.getInlineKind()) {
fn.setNoInline(*inlineKind == cir::InlineKind::NoInline);
fn.setInlineHint(*inlineKind == cir::InlineKind::InlineHint);
fn.setAlwaysInline(*inlineKind == cir::InlineKind::AlwaysInline);
}
if (std::optional<llvm::StringRef> personality = op.getPersonality())
fn.setPersonality(*personality);
fn.setVisibility_(
lowerCIRVisibilityToLLVMVisibility(op.getGlobalVisibility()));
rewriter.inlineRegionBefore(op.getBody(), fn.getBody(), fn.end());
if (failed(rewriter.convertRegionTypes(&fn.getBody(), *typeConverter,
&signatureConversion)))
return mlir::failure();
rewriter.eraseOp(op);
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMGetGlobalOpLowering::matchAndRewrite(
cir::GetGlobalOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// FIXME(cir): Premature DCE to avoid lowering stuff we're not using.
// CIRGen should mitigate this and not emit the get_global.
if (op->getUses().empty()) {
rewriter.eraseOp(op);
return mlir::success();
}
mlir::Type type = getTypeConverter()->convertType(op.getType());
mlir::Operation *newop = mlir::LLVM::AddressOfOp::create(
rewriter, op.getLoc(), type, op.getName());
if (op.getTls()) {
// Handle access to TLS via intrinsic.
newop = mlir::LLVM::ThreadlocalAddressOp::create(rewriter, op.getLoc(),
type, newop->getResult(0));
}
rewriter.replaceOp(op, newop);
return mlir::success();
}
llvm::SmallVector<mlir::NamedAttribute>
CIRToLLVMGlobalOpLowering::lowerGlobalAttributes(
cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const {
SmallVector<mlir::NamedAttribute> attributes;
if (mlir::StringAttr sectionAttr = op.getSectionAttr())
attributes.push_back(rewriter.getNamedAttr("section", sectionAttr));
mlir::LLVM::VisibilityAttr visibility = mlir::LLVM::VisibilityAttr::get(
getContext(),
lowerCIRVisibilityToLLVMVisibility(op.getGlobalVisibility()));
attributes.push_back(rewriter.getNamedAttr("visibility_", visibility));
if (op->getAttr(CUDAExternallyInitializedAttr::getMnemonic()))
attributes.push_back(rewriter.getNamedAttr("externally_initialized",
rewriter.getUnitAttr()));
return attributes;
}
/// Replace CIR global with a region initialized LLVM global and update
/// insertion point to the end of the initializer block.
void CIRToLLVMGlobalOpLowering::setupRegionInitializedLLVMGlobalOp(
cir::GlobalOp op, mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type llvmType =
convertTypeForMemory(*getTypeConverter(), dataLayout, op.getSymType());
// FIXME: These default values are placeholders until the the equivalent
// attributes are available on cir.global ops. This duplicates code
// in CIRToLLVMGlobalOpLowering::matchAndRewrite() but that will go
// away when the placeholders are no longer needed.
const bool isConst = op.getConstant();
unsigned addrSpace = 0;
if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
op.getAddrSpaceAttr()))
addrSpace = targetAS.getValue();
const bool isDsoLocal = op.getDsoLocal();
const bool isThreadLocal = (bool)op.getTlsModelAttr();
const uint64_t alignment = op.getAlignment().value_or(0);
const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
const StringRef symbol = op.getSymName();
mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
SmallVector<mlir::NamedAttribute> attributes =
lowerGlobalAttributes(op, rewriter);
mlir::LLVM::GlobalOp newGlobalOp =
rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
op, llvmType, isConst, linkage, symbol, nullptr, alignment, addrSpace,
isDsoLocal, isThreadLocal, comdatAttr, attributes);
newGlobalOp.getRegion().emplaceBlock();
rewriter.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
}
mlir::LogicalResult
CIRToLLVMGlobalOpLowering::matchAndRewriteRegionInitializedGlobal(
cir::GlobalOp op, mlir::Attribute init,
mlir::ConversionPatternRewriter &rewriter) const {
// TODO: Generalize this handling when more types are needed here.
assert((isa<cir::ConstArrayAttr, cir::ConstRecordAttr, cir::ConstVectorAttr,
cir::ConstPtrAttr, cir::ConstComplexAttr, cir::GlobalViewAttr,
cir::TypeInfoAttr, cir::UndefAttr, cir::PoisonAttr,
cir::VTableAttr, cir::ZeroAttr>(init)));
// TODO(cir): once LLVM's dialect has proper equivalent attributes this
// should be updated. For now, we use a custom op to initialize globals
// to the appropriate value.
const mlir::Location loc = op.getLoc();
setupRegionInitializedLLVMGlobalOp(op, rewriter);
CIRAttrToValue valueConverter(op, rewriter, typeConverter);
mlir::Value value = valueConverter.visit(init);
mlir::LLVM::ReturnOp::create(rewriter, loc, value);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMGlobalOpLowering::matchAndRewrite(
cir::GlobalOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// If this global requires non-trivial initialization or destruction,
// that needs to be moved to runtime handlers during LoweringPrepare.
if (!op.getCtorRegion().empty() || !op.getDtorRegion().empty())
return op.emitError() << "GlobalOp ctor and dtor regions should be removed "
"in LoweringPrepare";
std::optional<mlir::Attribute> init = op.getInitialValue();
// Fetch required values to create LLVM op.
const mlir::Type cirSymType = op.getSymType();
// This is the LLVM dialect type.
const mlir::Type llvmType =
convertTypeForMemory(*getTypeConverter(), dataLayout, cirSymType);
// FIXME: These default values are placeholders until the the equivalent
// attributes are available on cir.global ops.
const bool isConst = op.getConstant();
unsigned addrSpace = 0;
if (auto targetAS = mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
op.getAddrSpaceAttr()))
addrSpace = targetAS.getValue();
const bool isDsoLocal = op.getDsoLocal();
const bool isThreadLocal = (bool)op.getTlsModelAttr();
const uint64_t alignment = op.getAlignment().value_or(0);
const mlir::LLVM::Linkage linkage = convertLinkage(op.getLinkage());
const StringRef symbol = op.getSymName();
SmallVector<mlir::NamedAttribute> attributes =
lowerGlobalAttributes(op, rewriter);
if (init.has_value()) {
if (mlir::isa<cir::FPAttr, cir::IntAttr, cir::BoolAttr>(init.value())) {
GlobalInitAttrRewriter initRewriter(llvmType, rewriter);
init = initRewriter.visit(init.value());
// If initRewriter returned a null attribute, init will have a value but
// the value will be null. If that happens, initRewriter didn't handle the
// attribute type. It probably needs to be added to
// GlobalInitAttrRewriter.
if (!init.value()) {
op.emitError() << "unsupported initializer '" << init.value() << "'";
return mlir::failure();
}
} else if (mlir::isa<cir::ConstArrayAttr, cir::ConstVectorAttr,
cir::ConstRecordAttr, cir::ConstPtrAttr,
cir::ConstComplexAttr, cir::GlobalViewAttr,
cir::TypeInfoAttr, cir::UndefAttr, cir::PoisonAttr,
cir::VTableAttr, cir::ZeroAttr>(init.value())) {
// TODO(cir): once LLVM's dialect has proper equivalent attributes this
// should be updated. For now, we use a custom op to initialize globals
// to the appropriate value.
return matchAndRewriteRegionInitializedGlobal(op, init.value(), rewriter);
} else {
// We will only get here if new initializer types are added and this
// code is not updated to handle them.
op.emitError() << "unsupported initializer '" << init.value() << "'";
return mlir::failure();
}
}
mlir::SymbolRefAttr comdatAttr = getComdatAttr(op, rewriter);
rewriter.replaceOpWithNewOp<mlir::LLVM::GlobalOp>(
op, llvmType, isConst, linkage, symbol, init.value_or(mlir::Attribute()),
alignment, addrSpace, isDsoLocal, isThreadLocal, comdatAttr, attributes);
return mlir::success();
}
mlir::SymbolRefAttr
CIRToLLVMGlobalOpLowering::getComdatAttr(cir::GlobalOp &op,
mlir::OpBuilder &builder) const {
if (!op.getComdat())
return mlir::SymbolRefAttr{};
mlir::ModuleOp module = op->getParentOfType<mlir::ModuleOp>();
mlir::OpBuilder::InsertionGuard guard(builder);
StringRef comdatName("__llvm_comdat_globals");
if (!comdatOp) {
builder.setInsertionPointToStart(module.getBody());
comdatOp =
mlir::LLVM::ComdatOp::create(builder, module.getLoc(), comdatName);
}
if (auto comdatSelector = comdatOp.lookupSymbol<mlir::LLVM::ComdatSelectorOp>(
op.getSymName())) {
return mlir::SymbolRefAttr::get(
builder.getContext(), comdatName,
mlir::FlatSymbolRefAttr::get(comdatSelector.getSymNameAttr()));
}
builder.setInsertionPointToStart(&comdatOp.getBody().back());
auto selectorOp = mlir::LLVM::ComdatSelectorOp::create(
builder, comdatOp.getLoc(), op.getSymName(),
mlir::LLVM::comdat::Comdat::Any);
return mlir::SymbolRefAttr::get(
builder.getContext(), comdatName,
mlir::FlatSymbolRefAttr::get(selectorOp.getSymNameAttr()));
}
mlir::LogicalResult CIRToLLVMSwitchFlatOpLowering::matchAndRewrite(
cir::SwitchFlatOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
llvm::SmallVector<mlir::APInt, 8> caseValues;
for (mlir::Attribute val : op.getCaseValues()) {
auto intAttr = cast<cir::IntAttr>(val);
caseValues.push_back(intAttr.getValue());
}
llvm::SmallVector<mlir::Block *, 8> caseDestinations;
llvm::SmallVector<mlir::ValueRange, 8> caseOperands;
for (mlir::Block *x : op.getCaseDestinations())
caseDestinations.push_back(x);
for (mlir::OperandRange x : op.getCaseOperands())
caseOperands.push_back(x);
// Set switch op to branch to the newly created blocks.
rewriter.setInsertionPoint(op);
rewriter.replaceOpWithNewOp<mlir::LLVM::SwitchOp>(
op, adaptor.getCondition(), op.getDefaultDestination(),
op.getDefaultOperands(), caseValues, caseDestinations, caseOperands);
return mlir::success();
}
static mlir::LLVM::IntegerOverflowFlags nswFlag(bool nsw) {
return nsw ? mlir::LLVM::IntegerOverflowFlags::nsw
: mlir::LLVM::IntegerOverflowFlags::none;
}
template <typename CIROp, typename LLVMIntOp>
static mlir::LogicalResult
lowerIncDecOp(CIROp op, typename CIROp::Adaptor adaptor,
mlir::ConversionPatternRewriter &rewriter, double fpConstant) {
mlir::Type elementType = elementTypeIfVector(op.getType());
mlir::Type llvmType = adaptor.getInput().getType();
mlir::Location loc = op.getLoc();
if (mlir::isa<cir::IntType>(elementType)) {
auto maybeNSW = nswFlag(op.getNoSignedWrap());
auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1);
rewriter.replaceOpWithNewOp<LLVMIntOp>(op, adaptor.getInput(), one,
maybeNSW);
return mlir::success();
}
if (mlir::isa<cir::FPTypeInterface>(elementType)) {
auto fpConst = mlir::LLVM::ConstantOp::create(
rewriter, loc, rewriter.getFloatAttr(llvmType, fpConstant));
rewriter.replaceOpWithNewOp<mlir::LLVM::FAddOp>(op, fpConst,
adaptor.getInput());
return mlir::success();
}
return op.emitError() << "Unsupported type for IncOp/DecOp";
}
mlir::LogicalResult CIRToLLVMIncOpLowering::matchAndRewrite(
cir::IncOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerIncDecOp<cir::IncOp, mlir::LLVM::AddOp>(op, adaptor, rewriter,
1.0);
}
mlir::LogicalResult CIRToLLVMDecOpLowering::matchAndRewrite(
cir::DecOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerIncDecOp<cir::DecOp, mlir::LLVM::SubOp>(op, adaptor, rewriter,
-1.0);
}
mlir::LogicalResult CIRToLLVMMinusOpLowering::matchAndRewrite(
cir::MinusOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type elementType = elementTypeIfVector(op.getType());
bool isVector = mlir::isa<cir::VectorType>(op.getType());
mlir::Type llvmType = adaptor.getInput().getType();
mlir::Location loc = op.getLoc();
if (mlir::isa<cir::IntType>(elementType)) {
auto maybeNSW = nswFlag(op.getNoSignedWrap());
mlir::Value zero;
if (isVector)
zero = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmType);
else
zero = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 0);
rewriter.replaceOpWithNewOp<mlir::LLVM::SubOp>(op, zero, adaptor.getInput(),
maybeNSW);
return mlir::success();
}
if (mlir::isa<cir::FPTypeInterface>(elementType)) {
rewriter.replaceOpWithNewOp<mlir::LLVM::FNegOp>(op, adaptor.getInput());
return mlir::success();
}
return op.emitError() << "Unsupported type for unary minus";
}
mlir::LogicalResult CIRToLLVMNotOpLowering::matchAndRewrite(
cir::NotOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type elementType = elementTypeIfVector(op.getType());
bool isVector = mlir::isa<cir::VectorType>(op.getType());
mlir::Type llvmType = adaptor.getInput().getType();
mlir::Location loc = op.getLoc();
if (mlir::isa<cir::IntType>(elementType)) {
mlir::Value minusOne;
if (isVector) {
const uint64_t numElements =
mlir::dyn_cast<cir::VectorType>(op.getType()).getSize();
SmallVector<int32_t> values(numElements, -1);
mlir::DenseIntElementsAttr denseVec = rewriter.getI32VectorAttr(values);
minusOne =
mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, denseVec);
} else {
minusOne = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, -1);
}
rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(),
minusOne);
return mlir::success();
}
if (mlir::isa<cir::BoolType>(elementType)) {
auto one = mlir::LLVM::ConstantOp::create(rewriter, loc, llvmType, 1);
rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getInput(), one);
return mlir::success();
}
return op.emitError() << "Unsupported type for bitwise NOT";
}
static bool isIntTypeUnsigned(mlir::Type type) {
// TODO: Ideally, we should only need to check cir::IntType here.
return mlir::isa<cir::IntType>(type)
? mlir::cast<cir::IntType>(type).isUnsigned()
: mlir::cast<mlir::IntegerType>(type).isUnsigned();
}
//===----------------------------------------------------------------------===//
// Binary Op Lowering
//===----------------------------------------------------------------------===//
template <typename BinOp>
static mlir::LLVM::IntegerOverflowFlags intOverflowFlag(BinOp op) {
if (op.getNoUnsignedWrap())
return mlir::LLVM::IntegerOverflowFlags::nuw;
if (op.getNoSignedWrap())
return mlir::LLVM::IntegerOverflowFlags::nsw;
return mlir::LLVM::IntegerOverflowFlags::none;
}
/// Lower an arithmetic op that supports saturation, overflow flags, and an FP
/// variant. Used for Add and Sub which share identical dispatch logic.
template <typename UIntSatOp, typename SIntSatOp, typename IntOp, typename FPOp,
typename CIROp>
static mlir::LogicalResult
lowerSaturatableArithOp(CIROp op, mlir::Value lhs, mlir::Value rhs,
mlir::ConversionPatternRewriter &rewriter) {
const mlir::Type eltType = elementTypeIfVector(op.getRhs().getType());
if (cir::isIntOrBoolType(eltType)) {
if (op.getSaturated()) {
if (isIntTypeUnsigned(eltType))
rewriter.replaceOpWithNewOp<UIntSatOp>(op, lhs, rhs);
else
rewriter.replaceOpWithNewOp<SIntSatOp>(op, lhs, rhs);
return mlir::success();
}
rewriter.replaceOpWithNewOp<IntOp>(op, lhs, rhs, intOverflowFlag(op));
} else {
rewriter.replaceOpWithNewOp<FPOp>(op, lhs, rhs);
}
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAddOpLowering::matchAndRewrite(
cir::AddOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerSaturatableArithOp<mlir::LLVM::UAddSat, mlir::LLVM::SAddSat,
mlir::LLVM::AddOp, mlir::LLVM::FAddOp>(
op, adaptor.getLhs(), adaptor.getRhs(), rewriter);
}
mlir::LogicalResult CIRToLLVMSubOpLowering::matchAndRewrite(
cir::SubOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerSaturatableArithOp<mlir::LLVM::USubSat, mlir::LLVM::SSubSat,
mlir::LLVM::SubOp, mlir::LLVM::FSubOp>(
op, adaptor.getLhs(), adaptor.getRhs(), rewriter);
}
mlir::LogicalResult CIRToLLVMMulOpLowering::matchAndRewrite(
cir::MulOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Value lhs = adaptor.getLhs();
const mlir::Value rhs = adaptor.getRhs();
if (cir::isIntOrBoolType(elementTypeIfVector(op.getRhs().getType()))) {
rewriter.replaceOpWithNewOp<mlir::LLVM::MulOp>(op, lhs, rhs,
intOverflowFlag(op));
} else {
rewriter.replaceOpWithNewOp<mlir::LLVM::FMulOp>(op, lhs, rhs);
}
return mlir::success();
}
/// Lower a binary op that maps to unsigned/signed/FP LLVM ops depending on
/// operand type. Used for Div and Rem which share identical dispatch logic.
template <typename UIntOp, typename SIntOp, typename FPOp, typename CIROp>
static mlir::LogicalResult
lowerIntFPBinaryOp(CIROp op, mlir::Value lhs, mlir::Value rhs,
mlir::ConversionPatternRewriter &rewriter) {
const mlir::Type eltType = elementTypeIfVector(op.getRhs().getType());
if (cir::isIntOrBoolType(eltType)) {
if (isIntTypeUnsigned(eltType))
rewriter.replaceOpWithNewOp<UIntOp>(op, lhs, rhs);
else
rewriter.replaceOpWithNewOp<SIntOp>(op, lhs, rhs);
} else {
rewriter.replaceOpWithNewOp<FPOp>(op, lhs, rhs);
}
return mlir::success();
}
mlir::LogicalResult CIRToLLVMDivOpLowering::matchAndRewrite(
cir::DivOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerIntFPBinaryOp<mlir::LLVM::UDivOp, mlir::LLVM::SDivOp,
mlir::LLVM::FDivOp>(op, adaptor.getLhs(),
adaptor.getRhs(), rewriter);
}
mlir::LogicalResult CIRToLLVMRemOpLowering::matchAndRewrite(
cir::RemOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerIntFPBinaryOp<mlir::LLVM::URemOp, mlir::LLVM::SRemOp,
mlir::LLVM::FRemOp>(op, adaptor.getLhs(),
adaptor.getRhs(), rewriter);
}
mlir::LogicalResult CIRToLLVMAndOpLowering::matchAndRewrite(
cir::AndOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, adaptor.getLhs(),
adaptor.getRhs());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMOrOpLowering::matchAndRewrite(
cir::OrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, adaptor.getLhs(),
adaptor.getRhs());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMXorOpLowering::matchAndRewrite(
cir::XorOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::XOrOp>(op, adaptor.getLhs(),
adaptor.getRhs());
return mlir::success();
}
template <typename CIROp, typename UIntOp, typename SIntOp>
static mlir::LogicalResult
lowerMinMaxOp(CIROp op, typename CIROp::Adaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) {
const mlir::Value lhs = adaptor.getLhs();
const mlir::Value rhs = adaptor.getRhs();
if (isIntTypeUnsigned(elementTypeIfVector(op.getRhs().getType())))
rewriter.replaceOpWithNewOp<UIntOp>(op, lhs, rhs);
else
rewriter.replaceOpWithNewOp<SIntOp>(op, lhs, rhs);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMMaxOpLowering::matchAndRewrite(
cir::MaxOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerMinMaxOp<cir::MaxOp, mlir::LLVM::UMaxOp, mlir::LLVM::SMaxOp>(
op, adaptor, rewriter);
}
mlir::LogicalResult CIRToLLVMMinOpLowering::matchAndRewrite(
cir::MinOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerMinMaxOp<cir::MinOp, mlir::LLVM::UMinOp, mlir::LLVM::SMinOp>(
op, adaptor, rewriter);
}
/// Convert from a CIR comparison kind to an LLVM IR integral comparison kind.
static mlir::LLVM::ICmpPredicate
convertCmpKindToICmpPredicate(cir::CmpOpKind kind, bool isSigned) {
using CIR = cir::CmpOpKind;
using LLVMICmp = mlir::LLVM::ICmpPredicate;
switch (kind) {
case CIR::eq:
return LLVMICmp::eq;
case CIR::ne:
return LLVMICmp::ne;
case CIR::lt:
return (isSigned ? LLVMICmp::slt : LLVMICmp::ult);
case CIR::le:
return (isSigned ? LLVMICmp::sle : LLVMICmp::ule);
case CIR::gt:
return (isSigned ? LLVMICmp::sgt : LLVMICmp::ugt);
case CIR::ge:
return (isSigned ? LLVMICmp::sge : LLVMICmp::uge);
}
llvm_unreachable("Unknown CmpOpKind");
}
/// Convert from a CIR comparison kind to an LLVM IR floating-point comparison
/// kind.
static mlir::LLVM::FCmpPredicate
convertCmpKindToFCmpPredicate(cir::CmpOpKind kind) {
using CIR = cir::CmpOpKind;
using LLVMFCmp = mlir::LLVM::FCmpPredicate;
switch (kind) {
case CIR::eq:
return LLVMFCmp::oeq;
case CIR::ne:
return LLVMFCmp::une;
case CIR::lt:
return LLVMFCmp::olt;
case CIR::le:
return LLVMFCmp::ole;
case CIR::gt:
return LLVMFCmp::ogt;
case CIR::ge:
return LLVMFCmp::oge;
}
llvm_unreachable("Unknown CmpOpKind");
}
mlir::LogicalResult CIRToLLVMCmpOpLowering::matchAndRewrite(
cir::CmpOp cmpOp, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type type = cmpOp.getLhs().getType();
if (mlir::isa<cir::IntType, mlir::IntegerType>(type)) {
bool isSigned = mlir::isa<cir::IntType>(type)
? mlir::cast<cir::IntType>(type).isSigned()
: mlir::cast<mlir::IntegerType>(type).isSigned();
mlir::LLVM::ICmpPredicate kind =
convertCmpKindToICmpPredicate(cmpOp.getKind(), isSigned);
rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
return mlir::success();
}
if (auto ptrTy = mlir::dyn_cast<cir::PointerType>(type)) {
mlir::LLVM::ICmpPredicate kind =
convertCmpKindToICmpPredicate(cmpOp.getKind(),
/* isSigned=*/false);
rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
return mlir::success();
}
if (auto vptrTy = mlir::dyn_cast<cir::VPtrType>(type)) {
// !cir.vptr is a special case, but it's just a pointer to LLVM.
auto kind = convertCmpKindToICmpPredicate(cmpOp.getKind(),
/* isSigned=*/false);
rewriter.replaceOpWithNewOp<mlir::LLVM::ICmpOp>(
cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
return mlir::success();
}
if (mlir::isa<cir::FPTypeInterface>(type)) {
mlir::LLVM::FCmpPredicate kind =
convertCmpKindToFCmpPredicate(cmpOp.getKind());
rewriter.replaceOpWithNewOp<mlir::LLVM::FCmpOp>(
cmpOp, kind, adaptor.getLhs(), adaptor.getRhs());
return mlir::success();
}
if (mlir::isa<cir::ComplexType>(type)) {
mlir::Value lhs = adaptor.getLhs();
mlir::Value rhs = adaptor.getRhs();
mlir::Location loc = cmpOp.getLoc();
auto complexType = mlir::cast<cir::ComplexType>(cmpOp.getLhs().getType());
mlir::Type complexElemTy =
getTypeConverter()->convertType(complexType.getElementType());
auto lhsReal = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));
auto lhsImag = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));
auto rhsReal = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));
auto rhsImag = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));
if (cmpOp.getKind() == cir::CmpOpKind::eq) {
if (complexElemTy.isInteger()) {
auto realCmp = mlir::LLVM::ICmpOp::create(
rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsReal, rhsReal);
auto imagCmp = mlir::LLVM::ICmpOp::create(
rewriter, loc, mlir::LLVM::ICmpPredicate::eq, lhsImag, rhsImag);
rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
return mlir::success();
}
auto realCmp = mlir::LLVM::FCmpOp::create(
rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsReal, rhsReal);
auto imagCmp = mlir::LLVM::FCmpOp::create(
rewriter, loc, mlir::LLVM::FCmpPredicate::oeq, lhsImag, rhsImag);
rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(cmpOp, realCmp, imagCmp);
return mlir::success();
}
if (cmpOp.getKind() == cir::CmpOpKind::ne) {
if (complexElemTy.isInteger()) {
auto realCmp = mlir::LLVM::ICmpOp::create(
rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsReal, rhsReal);
auto imagCmp = mlir::LLVM::ICmpOp::create(
rewriter, loc, mlir::LLVM::ICmpPredicate::ne, lhsImag, rhsImag);
rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
return mlir::success();
}
auto realCmp = mlir::LLVM::FCmpOp::create(
rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsReal, rhsReal);
auto imagCmp = mlir::LLVM::FCmpOp::create(
rewriter, loc, mlir::LLVM::FCmpPredicate::une, lhsImag, rhsImag);
rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(cmpOp, realCmp, imagCmp);
return mlir::success();
}
}
return cmpOp.emitError() << "unsupported type for CmpOp: " << type;
}
/// Shared lowering logic for checked binary arithmetic overflow operations.
/// The \p opStr parameter specifies the arithmetic operation name used in the
/// LLVM intrinsic (e.g., "add", "sub", "mul").
template <typename OpTy>
static mlir::LogicalResult
lowerBinOpOverflow(OpTy op, typename OpTy::Adaptor adaptor,
mlir::ConversionPatternRewriter &rewriter,
const mlir::TypeConverter *typeConverter,
llvm::StringRef opStr) {
mlir::Location loc = op.getLoc();
cir::IntType operandTy = op.getLhs().getType();
cir::IntType resultTy = op.getResult().getType();
bool sign = operandTy.getIsSigned() || resultTy.getIsSigned();
unsigned width =
std::max(operandTy.getWidth() + (sign && operandTy.isUnsigned()),
resultTy.getWidth() + (sign && resultTy.isUnsigned()));
mlir::IntegerType encompassedLLVMTy = rewriter.getIntegerType(width);
mlir::Value lhs = adaptor.getLhs();
mlir::Value rhs = adaptor.getRhs();
if (operandTy.getWidth() < width) {
if (operandTy.isSigned()) {
lhs = mlir::LLVM::SExtOp::create(rewriter, loc, encompassedLLVMTy, lhs);
rhs = mlir::LLVM::SExtOp::create(rewriter, loc, encompassedLLVMTy, rhs);
} else {
lhs = mlir::LLVM::ZExtOp::create(rewriter, loc, encompassedLLVMTy, lhs);
rhs = mlir::LLVM::ZExtOp::create(rewriter, loc, encompassedLLVMTy, rhs);
}
}
// The intrinsic name is `@llvm.{s|u}{op}.with.overflow.i{width}`
std::string intrinName = ("llvm." + llvm::Twine(sign ? 's' : 'u') + opStr +
".with.overflow.i" + llvm::Twine(width))
.str();
auto intrinNameAttr = mlir::StringAttr::get(op.getContext(), intrinName);
mlir::IntegerType overflowLLVMTy = rewriter.getI1Type();
auto intrinRetTy = mlir::LLVM::LLVMStructType::getLiteral(
rewriter.getContext(), {encompassedLLVMTy, overflowLLVMTy});
auto callLLVMIntrinOp = mlir::LLVM::CallIntrinsicOp::create(
rewriter, loc, intrinRetTy, intrinNameAttr, mlir::ValueRange{lhs, rhs});
mlir::Value intrinRet = callLLVMIntrinOp.getResult(0);
mlir::Value result = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, intrinRet, ArrayRef<int64_t>{0})
.getResult();
mlir::Value overflow = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, intrinRet, ArrayRef<int64_t>{1})
.getResult();
if (resultTy.getWidth() < width) {
mlir::Type resultLLVMTy = typeConverter->convertType(resultTy);
auto truncResult =
mlir::LLVM::TruncOp::create(rewriter, loc, resultLLVMTy, result);
// Extend the truncated result back to the encompassing type to check for
// any overflows during the truncation.
mlir::Value truncResultExt;
if (resultTy.isSigned())
truncResultExt = mlir::LLVM::SExtOp::create(
rewriter, loc, encompassedLLVMTy, truncResult);
else
truncResultExt = mlir::LLVM::ZExtOp::create(
rewriter, loc, encompassedLLVMTy, truncResult);
auto truncOverflow = mlir::LLVM::ICmpOp::create(
rewriter, loc, mlir::LLVM::ICmpPredicate::ne, truncResultExt, result);
result = truncResult;
overflow = mlir::LLVM::OrOp::create(rewriter, loc, overflow, truncOverflow);
}
mlir::Type boolLLVMTy =
typeConverter->convertType(op.getOverflow().getType());
if (boolLLVMTy != rewriter.getI1Type())
overflow = mlir::LLVM::ZExtOp::create(rewriter, loc, boolLLVMTy, overflow);
rewriter.replaceOp(op, mlir::ValueRange{result, overflow});
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAddOverflowOpLowering::matchAndRewrite(
cir::AddOverflowOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "add");
}
mlir::LogicalResult CIRToLLVMSubOverflowOpLowering::matchAndRewrite(
cir::SubOverflowOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "sub");
}
mlir::LogicalResult CIRToLLVMMulOverflowOpLowering::matchAndRewrite(
cir::MulOverflowOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return lowerBinOpOverflow(op, adaptor, rewriter, getTypeConverter(), "mul");
}
mlir::LogicalResult CIRToLLVMShiftOpLowering::matchAndRewrite(
cir::ShiftOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
assert((op.getValue().getType() == op.getType()) &&
"inconsistent operands' types NYI");
const mlir::Type llvmTy = getTypeConverter()->convertType(op.getType());
mlir::Value amt = adaptor.getAmount();
mlir::Value val = adaptor.getValue();
auto cirAmtTy = mlir::dyn_cast<cir::IntType>(op.getAmount().getType());
bool isUnsigned;
if (cirAmtTy) {
auto cirValTy = mlir::cast<cir::IntType>(op.getValue().getType());
isUnsigned = cirValTy.isUnsigned();
// Ensure shift amount is the same type as the value. Some undefined
// behavior might occur in the casts below as per [C99 6.5.7.3].
// Vector type shift amount needs no cast as type consistency is expected to
// be already be enforced at CIRGen.
if (cirAmtTy)
amt = getLLVMIntCast(rewriter, amt, llvmTy, true, cirAmtTy.getWidth(),
cirValTy.getWidth());
} else {
auto cirValVTy = mlir::cast<cir::VectorType>(op.getValue().getType());
isUnsigned =
mlir::cast<cir::IntType>(cirValVTy.getElementType()).isUnsigned();
}
// Lower to the proper LLVM shift operation.
if (op.getIsShiftleft()) {
rewriter.replaceOpWithNewOp<mlir::LLVM::ShlOp>(op, llvmTy, val, amt);
return mlir::success();
}
if (isUnsigned)
rewriter.replaceOpWithNewOp<mlir::LLVM::LShrOp>(op, llvmTy, val, amt);
else
rewriter.replaceOpWithNewOp<mlir::LLVM::AShrOp>(op, llvmTy, val, amt);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMSelectOpLowering::matchAndRewrite(
cir::SelectOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto getConstantBool = [](mlir::Value value) -> cir::BoolAttr {
auto definingOp = value.getDefiningOp<cir::ConstantOp>();
if (!definingOp)
return {};
auto constValue = definingOp.getValueAttr<cir::BoolAttr>();
if (!constValue)
return {};
return constValue;
};
// Two special cases in the LLVMIR codegen of select op:
// - select %0, %1, false => and %0, %1
// - select %0, true, %1 => or %0, %1
if (mlir::isa<cir::BoolType>(op.getTrueValue().getType())) {
cir::BoolAttr trueValue = getConstantBool(op.getTrueValue());
cir::BoolAttr falseValue = getConstantBool(op.getFalseValue());
if (falseValue && !falseValue.getValue()) {
// select %0, %1, false => and %0, %1
rewriter.replaceOpWithNewOp<mlir::LLVM::AndOp>(op, adaptor.getCondition(),
adaptor.getTrueValue());
return mlir::success();
}
if (trueValue && trueValue.getValue()) {
// select %0, true, %1 => or %0, %1
rewriter.replaceOpWithNewOp<mlir::LLVM::OrOp>(op, adaptor.getCondition(),
adaptor.getFalseValue());
return mlir::success();
}
}
mlir::Value llvmCondition = adaptor.getCondition();
rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
op, llvmCondition, adaptor.getTrueValue(), adaptor.getFalseValue());
return mlir::success();
}
static void prepareTypeConverter(mlir::LLVMTypeConverter &converter,
mlir::DataLayout &dataLayout) {
converter.addConversion([&](cir::PointerType type) -> mlir::Type {
mlir::ptr::MemorySpaceAttrInterface addrSpaceAttr = type.getAddrSpace();
unsigned numericAS = 0;
if (auto targetAsAttr =
mlir::dyn_cast_if_present<cir::TargetAddressSpaceAttr>(
addrSpaceAttr))
numericAS = targetAsAttr.getValue();
return mlir::LLVM::LLVMPointerType::get(type.getContext(), numericAS);
});
converter.addConversion([&](cir::VPtrType type) -> mlir::Type {
assert(!cir::MissingFeatures::addressSpace());
return mlir::LLVM::LLVMPointerType::get(type.getContext());
});
converter.addConversion([&](cir::ArrayType type) -> mlir::Type {
mlir::Type ty =
convertTypeForMemory(converter, dataLayout, type.getElementType());
return mlir::LLVM::LLVMArrayType::get(ty, type.getSize());
});
converter.addConversion([&](cir::VectorType type) -> mlir::Type {
const mlir::Type ty = converter.convertType(type.getElementType());
return mlir::VectorType::get(type.getSize(), ty, {type.getIsScalable()});
});
converter.addConversion([&](cir::BoolType type) -> mlir::Type {
return mlir::IntegerType::get(type.getContext(), 1,
mlir::IntegerType::Signless);
});
converter.addConversion([&](cir::IntType type) -> mlir::Type {
// LLVM doesn't work with signed types, so we drop the CIR signs here.
return mlir::IntegerType::get(type.getContext(), type.getWidth());
});
converter.addConversion([&](cir::SingleType type) -> mlir::Type {
return mlir::Float32Type::get(type.getContext());
});
converter.addConversion([&](cir::DoubleType type) -> mlir::Type {
return mlir::Float64Type::get(type.getContext());
});
converter.addConversion([&](cir::FP80Type type) -> mlir::Type {
return mlir::Float80Type::get(type.getContext());
});
converter.addConversion([&](cir::FP128Type type) -> mlir::Type {
return mlir::Float128Type::get(type.getContext());
});
converter.addConversion([&](cir::LongDoubleType type) -> mlir::Type {
return converter.convertType(type.getUnderlying());
});
converter.addConversion([&](cir::FP16Type type) -> mlir::Type {
return mlir::Float16Type::get(type.getContext());
});
converter.addConversion([&](cir::BF16Type type) -> mlir::Type {
return mlir::BFloat16Type::get(type.getContext());
});
converter.addConversion([&](cir::ComplexType type) -> mlir::Type {
// A complex type is lowered to an LLVM struct that contains the real and
// imaginary part as data fields.
mlir::Type elementTy = converter.convertType(type.getElementType());
mlir::Type structFields[2] = {elementTy, elementTy};
return mlir::LLVM::LLVMStructType::getLiteral(type.getContext(),
structFields);
});
converter.addConversion([&](cir::FuncType type) -> std::optional<mlir::Type> {
auto result = converter.convertType(type.getReturnType());
llvm::SmallVector<mlir::Type> arguments;
arguments.reserve(type.getNumInputs());
if (converter.convertTypes(type.getInputs(), arguments).failed())
return std::nullopt;
auto varArg = type.isVarArg();
return mlir::LLVM::LLVMFunctionType::get(result, arguments, varArg);
});
converter.addConversion([&](cir::RecordType type) -> mlir::Type {
// Convert struct members.
llvm::SmallVector<mlir::Type> llvmMembers;
switch (type.getKind()) {
case cir::RecordType::Class:
case cir::RecordType::Struct:
for (mlir::Type ty : type.getMembers())
llvmMembers.push_back(convertTypeForMemory(converter, dataLayout, ty));
break;
// Unions are lowered as only the largest member.
case cir::RecordType::Union:
if (type.getMembers().empty())
break;
if (auto largestMember = type.getLargestMember(dataLayout))
llvmMembers.push_back(
convertTypeForMemory(converter, dataLayout, largestMember));
if (type.getPadded()) {
auto last = *type.getMembers().rbegin();
llvmMembers.push_back(
convertTypeForMemory(converter, dataLayout, last));
}
break;
}
// Record has a name: lower as an identified record.
mlir::LLVM::LLVMStructType llvmStruct;
if (type.getName()) {
llvmStruct = mlir::LLVM::LLVMStructType::getIdentified(
type.getContext(), type.getPrefixedName());
if (llvmStruct.setBody(llvmMembers, type.getPacked()).failed())
llvm_unreachable("Failed to set body of record");
} else { // Record has no name: lower as literal record.
llvmStruct = mlir::LLVM::LLVMStructType::getLiteral(
type.getContext(), llvmMembers, type.getPacked());
}
return llvmStruct;
});
converter.addConversion([&](cir::VoidType type) -> mlir::Type {
return mlir::LLVM::LLVMVoidType::get(type.getContext());
});
}
static void buildCtorDtorList(
mlir::ModuleOp module, StringRef globalXtorName, StringRef llvmXtorName,
llvm::function_ref<std::pair<StringRef, int>(mlir::Attribute)> createXtor) {
llvm::SmallVector<std::pair<StringRef, int>> globalXtors;
for (const mlir::NamedAttribute namedAttr : module->getAttrs()) {
if (namedAttr.getName() == globalXtorName) {
for (auto attr : mlir::cast<mlir::ArrayAttr>(namedAttr.getValue()))
globalXtors.emplace_back(createXtor(attr));
break;
}
}
if (globalXtors.empty())
return;
mlir::OpBuilder builder(module.getContext());
builder.setInsertionPointToEnd(&module.getBodyRegion().back());
// Create a global array llvm.global_ctors with element type of
// struct { i32, ptr, ptr }
auto ctorPFTy = mlir::LLVM::LLVMPointerType::get(builder.getContext());
llvm::SmallVector<mlir::Type> ctorStructFields;
ctorStructFields.push_back(builder.getI32Type());
ctorStructFields.push_back(ctorPFTy);
ctorStructFields.push_back(ctorPFTy);
auto ctorStructTy = mlir::LLVM::LLVMStructType::getLiteral(
builder.getContext(), ctorStructFields);
auto ctorStructArrayTy =
mlir::LLVM::LLVMArrayType::get(ctorStructTy, globalXtors.size());
mlir::Location loc = module.getLoc();
auto newGlobalOp = mlir::LLVM::GlobalOp::create(
builder, loc, ctorStructArrayTy, /*constant=*/false,
mlir::LLVM::Linkage::Appending, llvmXtorName, mlir::Attribute());
builder.createBlock(&newGlobalOp.getRegion());
builder.setInsertionPointToEnd(newGlobalOp.getInitializerBlock());
mlir::Value result =
mlir::LLVM::UndefOp::create(builder, loc, ctorStructArrayTy);
for (auto [index, fn] : llvm::enumerate(globalXtors)) {
mlir::Value structInit =
mlir::LLVM::UndefOp::create(builder, loc, ctorStructTy);
mlir::Value initPriority = mlir::LLVM::ConstantOp::create(
builder, loc, ctorStructFields[0], fn.second);
mlir::Value initFuncAddr = mlir::LLVM::AddressOfOp::create(
builder, loc, ctorStructFields[1], fn.first);
mlir::Value initAssociate =
mlir::LLVM::ZeroOp::create(builder, loc, ctorStructFields[2]);
// Literal zero makes the InsertValueOp::create ambiguous.
llvm::SmallVector<int64_t> zero{0};
structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,
initPriority, zero);
structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,
initFuncAddr, 1);
// TODO: handle associated data for initializers.
structInit = mlir::LLVM::InsertValueOp::create(builder, loc, structInit,
initAssociate, 2);
result = mlir::LLVM::InsertValueOp::create(builder, loc, result, structInit,
index);
}
mlir::LLVM::ReturnOp::create(builder, loc, result);
}
// The applyPartialConversion function traverses blocks in the dominance order,
// so it does not lower and operations that are not reachachable from the
// operations passed in as arguments. Since we do need to lower such code in
// order to avoid verification errors occur, we cannot just pass the module op
// to applyPartialConversion. We must build a set of unreachable ops and
// explicitly add them, along with the module, to the vector we pass to
// applyPartialConversion.
//
// For instance, this CIR code:
//
// cir.func @foo(%arg0: !s32i) -> !s32i {
// %4 = cir.cast int_to_bool %arg0 : !s32i -> !cir.bool
// cir.if %4 {
// %5 = cir.const #cir.int<1> : !s32i
// cir.return %5 : !s32i
// } else {
// %5 = cir.const #cir.int<0> : !s32i
// cir.return %5 : !s32i
// }
// cir.return %arg0 : !s32i
// }
//
// contains an unreachable return operation (the last one). After the flattening
// pass it will be placed into the unreachable block. The possible error
// after the lowering pass is: error: 'cir.return' op expects parent op to be
// one of 'cir.func, cir.scope, cir.if ... The reason that this operation was
// not lowered and the new parent is llvm.func.
//
// In the future we may want to get rid of this function and use a DCE pass or
// something similar. But for now we need to guarantee the absence of the
// dialect verification errors.
static void collectUnreachable(mlir::Operation *parent,
llvm::SmallVector<mlir::Operation *> &ops) {
llvm::SmallVector<mlir::Block *> unreachableBlocks;
parent->walk([&](mlir::Block *blk) { // check
if (blk->hasNoPredecessors() && !blk->isEntryBlock())
unreachableBlocks.push_back(blk);
});
std::set<mlir::Block *> visited;
for (mlir::Block *root : unreachableBlocks) {
// We create a work list for each unreachable block.
// Thus we traverse operations in some order.
std::deque<mlir::Block *> workList;
workList.push_back(root);
while (!workList.empty()) {
mlir::Block *blk = workList.back();
workList.pop_back();
if (visited.count(blk))
continue;
visited.emplace(blk);
for (mlir::Operation &op : *blk)
ops.push_back(&op);
for (mlir::Block *succ : blk->getSuccessors())
workList.push_back(succ);
}
}
}
mlir::LogicalResult CIRToLLVMObjSizeOpLowering::matchAndRewrite(
cir::ObjSizeOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type llvmResTy = getTypeConverter()->convertType(op.getType());
mlir::Location loc = op->getLoc();
mlir::IntegerType i1Ty = rewriter.getI1Type();
auto i1Val = [&rewriter, &loc, &i1Ty](bool val) {
return mlir::LLVM::ConstantOp::create(rewriter, loc, i1Ty, val);
};
replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.objectsize", llvmResTy,
{
adaptor.getPtr(),
i1Val(op.getMin()),
i1Val(op.getNullunknown()),
i1Val(op.getDynamic()),
});
return mlir::LogicalResult::success();
}
void ConvertCIRToLLVMPass::resolveBlockAddressOp(
LLVMBlockAddressInfo &blockInfoAddr) {
mlir::ModuleOp module = getOperation();
mlir::OpBuilder opBuilder(module.getContext());
for (auto &[blockAddOp, blockInfo] :
blockInfoAddr.getUnresolvedBlockAddress()) {
mlir::LLVM::BlockTagOp resolvedLabel =
blockInfoAddr.lookupBlockTag(blockInfo);
assert(resolvedLabel && "expected BlockTagOp to already be emitted");
mlir::FlatSymbolRefAttr fnSym = blockInfo.getFunc();
auto blkAddTag = mlir::LLVM::BlockAddressAttr::get(
opBuilder.getContext(), fnSym, resolvedLabel.getTagAttr());
blockAddOp.setBlockAddrAttr(blkAddTag);
}
blockInfoAddr.clearUnresolvedMap();
}
void ConvertCIRToLLVMPass::processCIRAttrs(mlir::ModuleOp module) {
// Lower the module attributes to LLVM equivalents.
if (mlir::Attribute tripleAttr =
module->getAttr(cir::CIRDialect::getTripleAttrName()))
module->setAttr(mlir::LLVM::LLVMDialect::getTargetTripleAttrName(),
tripleAttr);
if (mlir::Attribute asmAttr =
module->getAttr(cir::CIRDialect::getModuleLevelAsmAttrName()))
module->setAttr(mlir::LLVM::LLVMDialect::getModuleLevelAsmAttrName(),
asmAttr);
}
void ConvertCIRToLLVMPass::runOnOperation() {
llvm::TimeTraceScope scope("Convert CIR to LLVM Pass");
mlir::ModuleOp module = getOperation();
mlir::DataLayout dl(module);
mlir::LLVMTypeConverter converter(&getContext());
prepareTypeConverter(converter, dl);
/// Tracks the state required to lower CIR `LabelOp` and `BlockAddressOp`.
/// Maps labels to their corresponding `BlockTagOp` and keeps bookkeeping
/// of unresolved `BlockAddressOp`s until they are matched with the
/// corresponding `BlockTagOp` in `resolveBlockAddressOp`.
LLVMBlockAddressInfo blockInfoAddr;
mlir::RewritePatternSet patterns(&getContext());
patterns.add<CIRToLLVMBlockAddressOpLowering, CIRToLLVMLabelOpLowering>(
converter, patterns.getContext(), dl, blockInfoAddr);
patterns.add<
#define GET_LLVM_LOWERING_PATTERNS_LIST
#include "clang/CIR/Dialect/IR/CIRLowering.inc"
#undef GET_LLVM_LOWERING_PATTERNS_LIST
>(converter, patterns.getContext(), dl);
processCIRAttrs(module);
mlir::ConversionTarget target(getContext());
target.addLegalOp<mlir::ModuleOp>();
target.addLegalDialect<mlir::LLVM::LLVMDialect>();
target.addIllegalDialect<mlir::BuiltinDialect, cir::CIRDialect,
mlir::func::FuncDialect>();
llvm::SmallVector<mlir::Operation *> ops;
ops.push_back(module);
collectUnreachable(module, ops);
if (failed(applyPartialConversion(ops, target, std::move(patterns))))
signalPassFailure();
// Emit the llvm.global_ctors array.
buildCtorDtorList(module, cir::CIRDialect::getGlobalCtorsAttrName(),
"llvm.global_ctors", [](mlir::Attribute attr) {
auto ctorAttr = mlir::cast<cir::GlobalCtorAttr>(attr);
return std::make_pair(ctorAttr.getName(),
ctorAttr.getPriority());
});
// Emit the llvm.global_dtors array.
buildCtorDtorList(module, cir::CIRDialect::getGlobalDtorsAttrName(),
"llvm.global_dtors", [](mlir::Attribute attr) {
auto dtorAttr = mlir::cast<cir::GlobalDtorAttr>(attr);
return std::make_pair(dtorAttr.getName(),
dtorAttr.getPriority());
});
resolveBlockAddressOp(blockInfoAddr);
}
mlir::LogicalResult CIRToLLVMBrOpLowering::matchAndRewrite(
cir::BrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::BrOp>(op, adaptor.getOperands(),
op.getDest());
return mlir::LogicalResult::success();
}
mlir::LogicalResult CIRToLLVMGetMemberOpLowering::matchAndRewrite(
cir::GetMemberOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type llResTy = getTypeConverter()->convertType(op.getType());
const auto recordTy =
mlir::cast<cir::RecordType>(op.getAddrTy().getPointee());
assert(recordTy && "expected record type");
switch (recordTy.getKind()) {
case cir::RecordType::Class:
case cir::RecordType::Struct: {
// Since the base address is a pointer to an aggregate, the first offset
// is always zero. The second offset tell us which member it will access.
llvm::SmallVector<mlir::LLVM::GEPArg, 2> offset{0, op.getIndex()};
const mlir::Type elementTy = getTypeConverter()->convertType(recordTy);
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, llResTy, elementTy,
adaptor.getAddr(), offset);
return mlir::success();
}
case cir::RecordType::Union:
// Union members share the address space, so we just need a bitcast to
// conform to type-checking.
rewriter.replaceOpWithNewOp<mlir::LLVM::BitcastOp>(op, llResTy,
adaptor.getAddr());
return mlir::success();
}
}
mlir::LogicalResult CIRToLLVMExtractMemberOpLowering::matchAndRewrite(
cir::ExtractMemberOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
std::int64_t indices[1] = {static_cast<std::int64_t>(op.getIndex())};
mlir::Type recordTy = op.getRecord().getType();
auto cirRecordTy = mlir::cast<cir::RecordType>(recordTy);
switch (cirRecordTy.getKind()) {
case cir::RecordType::Struct:
case cir::RecordType::Class:
rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractValueOp>(
op, adaptor.getRecord(), indices);
return mlir::success();
case cir::RecordType::Union:
op.emitError("cir.extract_member cannot extract member from a union");
return mlir::failure();
}
llvm_unreachable("Unexpected record kind");
}
mlir::LogicalResult CIRToLLVMInsertMemberOpLowering::matchAndRewrite(
cir::InsertMemberOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
std::int64_t indecies[1] = {static_cast<std::int64_t>(op.getIndex())};
mlir::Type recordTy = op.getRecord().getType();
if (auto cirRecordTy = mlir::dyn_cast<cir::RecordType>(recordTy)) {
if (cirRecordTy.getKind() == cir::RecordType::Union) {
op.emitError("cir.update_member cannot update member of a union");
return mlir::failure();
}
}
rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
op, adaptor.getRecord(), adaptor.getValue(), indecies);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMUnreachableOpLowering::matchAndRewrite(
cir::UnreachableOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::UnreachableOp>(op);
return mlir::success();
}
void createLLVMFuncOpIfNotExist(mlir::ConversionPatternRewriter &rewriter,
mlir::Operation *srcOp, llvm::StringRef fnName,
mlir::Type fnTy) {
auto modOp = srcOp->getParentOfType<mlir::ModuleOp>();
auto enclosingFnOp = srcOp->getParentOfType<mlir::LLVM::LLVMFuncOp>();
mlir::Operation *sourceSymbol =
mlir::SymbolTable::lookupSymbolIn(modOp, fnName);
if (!sourceSymbol) {
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPoint(enclosingFnOp);
mlir::LLVM::LLVMFuncOp::create(rewriter, srcOp->getLoc(), fnName, fnTy);
}
}
mlir::LogicalResult CIRToLLVMThrowOpLowering::matchAndRewrite(
cir::ThrowOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Location loc = op.getLoc();
auto voidTy = mlir::LLVM::LLVMVoidType::get(getContext());
if (op.rethrows()) {
auto funcTy = mlir::LLVM::LLVMFunctionType::get(voidTy, {});
// Get or create `declare void @__cxa_rethrow()`
const llvm::StringRef functionName = "__cxa_rethrow";
createLLVMFuncOpIfNotExist(rewriter, op, functionName, funcTy);
auto cxaRethrow = mlir::LLVM::CallOp::create(
rewriter, loc, mlir::TypeRange{}, functionName);
rewriter.replaceOp(op, cxaRethrow);
return mlir::success();
}
auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
auto fnTy = mlir::LLVM::LLVMFunctionType::get(
voidTy, {llvmPtrTy, llvmPtrTy, llvmPtrTy});
// Get or create `declare void @__cxa_throw(ptr, ptr, ptr)`
const llvm::StringRef fnName = "__cxa_throw";
createLLVMFuncOpIfNotExist(rewriter, op, fnName, fnTy);
mlir::Value typeInfo = mlir::LLVM::AddressOfOp::create(
rewriter, loc, mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),
adaptor.getTypeInfoAttr());
mlir::Value dtor;
if (op.getDtor()) {
dtor = mlir::LLVM::AddressOfOp::create(rewriter, loc, llvmPtrTy,
adaptor.getDtorAttr());
} else {
dtor = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy);
}
auto cxaThrowCall = mlir::LLVM::CallOp::create(
rewriter, loc, mlir::TypeRange{}, fnName,
mlir::ValueRange{adaptor.getExceptionPtr(), typeInfo, dtor});
rewriter.replaceOp(op, cxaThrowCall);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAllocExceptionOpLowering::matchAndRewrite(
cir::AllocExceptionOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// Get or create `declare ptr @__cxa_allocate_exception(i64)`
StringRef fnName = "__cxa_allocate_exception";
auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
auto int64Ty = mlir::IntegerType::get(rewriter.getContext(), 64);
auto fnTy = mlir::LLVM::LLVMFunctionType::get(llvmPtrTy, {int64Ty});
createLLVMFuncOpIfNotExist(rewriter, op, fnName, fnTy);
auto exceptionSize = mlir::LLVM::ConstantOp::create(rewriter, op.getLoc(),
adaptor.getSizeAttr());
auto allocaExceptionCall = mlir::LLVM::CallOp::create(
rewriter, op.getLoc(), mlir::TypeRange{llvmPtrTy}, fnName,
mlir::ValueRange{exceptionSize});
rewriter.replaceOp(op, allocaExceptionCall);
return mlir::success();
}
static mlir::LLVM::LLVMStructType
getLLVMLandingPadStructTy(mlir::ConversionPatternRewriter &rewriter) {
// Create the landing pad type: struct { ptr, i32 }
mlir::MLIRContext *ctx = rewriter.getContext();
auto llvmPtr = mlir::LLVM::LLVMPointerType::get(ctx);
llvm::SmallVector<mlir::Type> structFields = {llvmPtr, rewriter.getI32Type()};
return mlir::LLVM::LLVMStructType::getLiteral(ctx, structFields);
}
mlir::LogicalResult CIRToLLVMEhInflightOpLowering::matchAndRewrite(
cir::EhInflightOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto llvmFn = op->getParentOfType<mlir::LLVM::LLVMFuncOp>();
assert(llvmFn && "expected LLVM function parent");
mlir::Block *entryBlock = &llvmFn.getRegion().front();
assert(entryBlock->isEntryBlock());
mlir::ArrayAttr catchListAttr = op.getCatchTypeListAttr();
mlir::SmallVector<mlir::Value> catchSymAddrs;
auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
mlir::Location loc = op.getLoc();
// %landingpad = landingpad { ptr, i32 }
// Note that since llvm.landingpad has to be the first operation on the
// block, any needed value for its operands has to be added somewhere else.
if (catchListAttr) {
// catch ptr @_ZTIi
// catch ptr @_ZTIPKc
for (mlir::Attribute catchAttr : catchListAttr) {
auto symAttr = cast<mlir::FlatSymbolRefAttr>(catchAttr);
// Generate `llvm.mlir.addressof` for each symbol, and place those
// operations in the LLVM function entry basic block.
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPointToStart(entryBlock);
mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
rewriter, loc, llvmPtrTy, symAttr.getValue());
catchSymAddrs.push_back(addrOp);
}
}
// Emit a catch-all clause (catch ptr null) when:
// - The catch_all attribute is set (typed catches + catch-all), or
// - No typed catches and no cleanup (legacy pure catch-all form)
if (op.getCatchAll() || (!catchListAttr && !op.getCleanup())) {
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPointToStart(entryBlock);
mlir::Value nullOp = mlir::LLVM::ZeroOp::create(rewriter, loc, llvmPtrTy);
catchSymAddrs.push_back(nullOp);
}
// %slot = extractvalue { ptr, i32 } %x, 0
// %selector = extractvalue { ptr, i32 } %x, 1
mlir::LLVM::LLVMStructType llvmLandingPadStructTy =
getLLVMLandingPadStructTy(rewriter);
auto landingPadOp = mlir::LLVM::LandingpadOp::create(
rewriter, loc, llvmLandingPadStructTy, catchSymAddrs);
// The LLVM cleanup flag is only needed when there is no catch-all handler,
// since catch-all (catch ptr null) already ensures the personality function
// enters the landing pad for all exception types.
if (op.getCleanup() && !op.getCatchAll())
landingPadOp.setCleanup(true);
mlir::Value slot =
mlir::LLVM::ExtractValueOp::create(rewriter, loc, landingPadOp, 0);
mlir::Value selector =
mlir::LLVM::ExtractValueOp::create(rewriter, loc, landingPadOp, 1);
rewriter.replaceOp(op, mlir::ValueRange{slot, selector});
return mlir::success();
}
mlir::LogicalResult CIRToLLVMResumeFlatOpLowering::matchAndRewrite(
cir::ResumeFlatOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// %lpad.val = insertvalue { ptr, i32 } poison, ptr %exception_ptr, 0
// %lpad.val2 = insertvalue { ptr, i32 } %lpad.val, i32 %selector, 1
// resume { ptr, i32 } %lpad.val2
mlir::Type llvmLandingPadStructTy = getLLVMLandingPadStructTy(rewriter);
mlir::Value poison = mlir::LLVM::PoisonOp::create(rewriter, op.getLoc(),
llvmLandingPadStructTy);
SmallVector<int64_t> slotIdx = {0};
mlir::Value slot = mlir::LLVM::InsertValueOp::create(
rewriter, op.getLoc(), poison, adaptor.getExceptionPtr(), slotIdx);
SmallVector<int64_t> selectorIdx = {1};
mlir::Value selector = mlir::LLVM::InsertValueOp::create(
rewriter, op.getLoc(), slot, adaptor.getTypeId(), selectorIdx);
rewriter.replaceOpWithNewOp<mlir::LLVM::ResumeOp>(op, selector);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMEhTypeIdOpLowering::matchAndRewrite(
cir::EhTypeIdOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Value addrOp = mlir::LLVM::AddressOfOp::create(
rewriter, op.getLoc(),
mlir::LLVM::LLVMPointerType::get(rewriter.getContext()),
op.getTypeSymAttr());
rewriter.replaceOpWithNewOp<mlir::LLVM::EhTypeidForOp>(
op, rewriter.getI32Type(), addrOp);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMEhSetjmpOpLowering::matchAndRewrite(
cir::EhSetjmpOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type returnType = typeConverter->convertType(op.getType());
mlir::LLVM::CallIntrinsicOp newOp =
createCallLLVMIntrinsicOp(rewriter, op.getLoc(), "llvm.eh.sjlj.setjmp",
returnType, adaptor.getEnv());
rewriter.replaceOp(op, newOp);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMEhLongjmpOpLowering::matchAndRewrite(
cir::EhLongjmpOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
replaceOpWithCallLLVMIntrinsicOp(rewriter, op, "llvm.eh.sjlj.longjmp",
/*resultTy=*/{}, adaptor.getOperands());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMTrapOpLowering::matchAndRewrite(
cir::TrapOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Location loc = op->getLoc();
rewriter.eraseOp(op);
mlir::LLVM::Trap::create(rewriter, loc);
// Note that the call to llvm.trap is not a terminator in LLVM dialect.
// So we must emit an additional llvm.unreachable to terminate the current
// block.
mlir::LLVM::UnreachableOp::create(rewriter, loc);
return mlir::success();
}
static mlir::Value
getValueForVTableSymbol(mlir::Operation *op,
mlir::ConversionPatternRewriter &rewriter,
const mlir::TypeConverter *converter,
mlir::FlatSymbolRefAttr nameAttr, mlir::Type &eltType) {
auto module = op->getParentOfType<mlir::ModuleOp>();
mlir::Operation *symbol = mlir::SymbolTable::lookupSymbolIn(module, nameAttr);
if (auto llvmSymbol = mlir::dyn_cast<mlir::LLVM::GlobalOp>(symbol)) {
eltType = llvmSymbol.getType();
} else if (auto cirSymbol = mlir::dyn_cast<cir::GlobalOp>(symbol)) {
eltType = converter->convertType(cirSymbol.getSymType());
} else {
op->emitError() << "unexpected symbol type for " << symbol;
return {};
}
return mlir::LLVM::AddressOfOp::create(
rewriter, op->getLoc(),
mlir::LLVM::LLVMPointerType::get(op->getContext()), nameAttr.getValue());
}
mlir::LogicalResult CIRToLLVMVTableAddrPointOpLowering::matchAndRewrite(
cir::VTableAddrPointOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::TypeConverter *converter = getTypeConverter();
mlir::Type targetType = converter->convertType(op.getType());
llvm::SmallVector<mlir::LLVM::GEPArg> offsets;
mlir::Type eltType;
mlir::Value symAddr = getValueForVTableSymbol(op, rewriter, converter,
op.getNameAttr(), eltType);
if (!symAddr)
return op.emitError() << "Unable to get value for vtable symbol";
offsets = llvm::SmallVector<mlir::LLVM::GEPArg>{
0, op.getAddressPointAttr().getIndex(),
op.getAddressPointAttr().getOffset()};
assert(eltType && "Shouldn't ever be missing an eltType here");
mlir::LLVM::GEPNoWrapFlags inboundsNuw =
mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(op, targetType, eltType,
symAddr, offsets, inboundsNuw);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVTableGetVPtrOpLowering::matchAndRewrite(
cir::VTableGetVPtrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// cir.vtable.get_vptr is equivalent to a bitcast from the source object
// pointer to the vptr type. Since the LLVM dialect uses opaque pointers
// we can just replace uses of this operation with the original pointer.
mlir::Value srcVal = adaptor.getSrc();
rewriter.replaceOp(op, srcVal);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVTableGetVirtualFnAddrOpLowering::matchAndRewrite(
cir::VTableGetVirtualFnAddrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type targetType = getTypeConverter()->convertType(op.getType());
auto eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
llvm::SmallVector<mlir::LLVM::GEPArg> offsets =
llvm::SmallVector<mlir::LLVM::GEPArg>{op.getIndex()};
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
op, targetType, eltType, adaptor.getVptr(), offsets,
mlir::LLVM::GEPNoWrapFlags::inbounds);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVTTAddrPointOpLowering::matchAndRewrite(
cir::VTTAddrPointOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type resultType = getTypeConverter()->convertType(op.getType());
llvm::SmallVector<mlir::LLVM::GEPArg> offsets;
mlir::Type eltType;
mlir::Value llvmAddr = adaptor.getSymAddr();
if (op.getSymAddr()) {
if (op.getOffset() == 0) {
rewriter.replaceOp(op, {llvmAddr});
return mlir::success();
}
offsets.push_back(adaptor.getOffset());
eltType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
} else {
llvmAddr = getValueForVTableSymbol(op, rewriter, getTypeConverter(),
op.getNameAttr(), eltType);
assert(eltType && "Shouldn't ever be missing an eltType here");
offsets.push_back(0);
offsets.push_back(adaptor.getOffset());
}
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
op, resultType, eltType, llvmAddr, offsets,
mlir::LLVM::GEPNoWrapFlags::inbounds);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMStackSaveOpLowering::matchAndRewrite(
cir::StackSaveOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
const mlir::Type ptrTy = getTypeConverter()->convertType(op.getType());
rewriter.replaceOpWithNewOp<mlir::LLVM::StackSaveOp>(op, ptrTy);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMStackRestoreOpLowering::matchAndRewrite(
cir::StackRestoreOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::StackRestoreOp>(op, adaptor.getPtr());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVecCreateOpLowering::matchAndRewrite(
cir::VecCreateOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// Start with an 'undef' value for the vector. Then 'insertelement' for
// each of the vector elements.
const auto vecTy = mlir::cast<cir::VectorType>(op.getType());
const mlir::Type llvmTy = typeConverter->convertType(vecTy);
const mlir::Location loc = op.getLoc();
mlir::Value result = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy);
assert(vecTy.getSize() == op.getElements().size() &&
"cir.vec.create op count doesn't match vector type elements count");
for (uint64_t i = 0; i < vecTy.getSize(); ++i) {
const mlir::Value indexValue =
mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);
result = mlir::LLVM::InsertElementOp::create(
rewriter, loc, result, adaptor.getElements()[i], indexValue);
}
rewriter.replaceOp(op, result);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVecExtractOpLowering::matchAndRewrite(
cir::VecExtractOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::ExtractElementOp>(
op, adaptor.getVec(), adaptor.getIndex());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVecInsertOpLowering::matchAndRewrite(
cir::VecInsertOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::InsertElementOp>(
op, adaptor.getVec(), adaptor.getValue(), adaptor.getIndex());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVecCmpOpLowering::matchAndRewrite(
cir::VecCmpOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type elementType = elementTypeIfVector(op.getLhs().getType());
mlir::Value bitResult;
if (auto intType = mlir::dyn_cast<cir::IntType>(elementType)) {
bitResult = mlir::LLVM::ICmpOp::create(
rewriter, op.getLoc(),
convertCmpKindToICmpPredicate(op.getKind(), intType.isSigned()),
adaptor.getLhs(), adaptor.getRhs());
} else if (mlir::isa<cir::FPTypeInterface>(elementType)) {
bitResult = mlir::LLVM::FCmpOp::create(
rewriter, op.getLoc(), convertCmpKindToFCmpPredicate(op.getKind()),
adaptor.getLhs(), adaptor.getRhs());
} else {
return op.emitError() << "unsupported type for VecCmpOp: " << elementType;
}
// LLVM IR vector comparison returns a vector of i1. This one-bit vector
// must be sign-extended to the correct result type, unless a vector of i1 is
// the type we need.
if (cast<cir::IntType>(cast<cir::VectorType>(op.getType()).getElementType())
.getWidth() > 1)
rewriter.replaceOpWithNewOp<mlir::LLVM::SExtOp>(
op, typeConverter->convertType(op.getType()), bitResult);
else
rewriter.replaceOp(op, bitResult);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVecSplatOpLowering::matchAndRewrite(
cir::VecSplatOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// Vector splat can be implemented with an `insertelement` and a
// `shufflevector`, which is better than an `insertelement` for each
// element in the vector. Start with an undef vector. Insert the value into
// the first element. Then use a `shufflevector` with a mask of all 0 to
// fill out the entire vector with that value.
cir::VectorType vecTy = op.getType();
mlir::Type llvmTy = typeConverter->convertType(vecTy);
mlir::Location loc = op.getLoc();
mlir::Value poison = mlir::LLVM::PoisonOp::create(rewriter, loc, llvmTy);
mlir::Value elementValue = adaptor.getValue();
if (elementValue.getDefiningOp<mlir::LLVM::PoisonOp>()) {
// If the splat value is poison, then we can just use poison value
// for the entire vector.
rewriter.replaceOp(op, poison);
return mlir::success();
}
if (auto constValue = elementValue.getDefiningOp<mlir::LLVM::ConstantOp>()) {
if (auto intAttr = dyn_cast<mlir::IntegerAttr>(constValue.getValue())) {
mlir::DenseIntElementsAttr denseVec = mlir::DenseIntElementsAttr::get(
mlir::cast<mlir::ShapedType>(llvmTy), intAttr.getValue());
rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
op, denseVec.getType(), denseVec);
return mlir::success();
}
if (auto fpAttr = dyn_cast<mlir::FloatAttr>(constValue.getValue())) {
mlir::DenseFPElementsAttr denseVec = mlir::DenseFPElementsAttr::get(
mlir::cast<mlir::ShapedType>(llvmTy), fpAttr.getValue());
rewriter.replaceOpWithNewOp<mlir::LLVM::ConstantOp>(
op, denseVec.getType(), denseVec);
return mlir::success();
}
}
mlir::Value indexValue =
mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), 0);
mlir::Value oneElement = mlir::LLVM::InsertElementOp::create(
rewriter, loc, poison, elementValue, indexValue);
SmallVector<int32_t> zeroValues(vecTy.getSize(), 0);
rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(op, oneElement,
poison, zeroValues);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVecShuffleOpLowering::matchAndRewrite(
cir::VecShuffleOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// LLVM::ShuffleVectorOp takes an ArrayRef of int for the list of indices.
// Convert the ClangIR ArrayAttr of IntAttr constants into a
// SmallVector<int>.
SmallVector<int, 8> indices;
std::transform(
op.getIndices().begin(), op.getIndices().end(),
std::back_inserter(indices), [](mlir::Attribute intAttr) {
return mlir::cast<cir::IntAttr>(intAttr).getValue().getSExtValue();
});
rewriter.replaceOpWithNewOp<mlir::LLVM::ShuffleVectorOp>(
op, adaptor.getVec1(), adaptor.getVec2(), indices);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVecShuffleDynamicOpLowering::matchAndRewrite(
cir::VecShuffleDynamicOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// LLVM IR does not have an operation that corresponds to this form of
// the built-in.
// __builtin_shufflevector(V, I)
// is implemented as this pseudocode, where the for loop is unrolled
// and N is the number of elements:
//
// result = undef
// maskbits = NextPowerOf2(N - 1)
// masked = I & maskbits
// for (i in 0 <= i < N)
// result[i] = V[masked[i]]
mlir::Location loc = op.getLoc();
mlir::Value input = adaptor.getVec();
mlir::Type llvmIndexVecType =
getTypeConverter()->convertType(op.getIndices().getType());
mlir::Type llvmIndexType = getTypeConverter()->convertType(
elementTypeIfVector(op.getIndices().getType()));
uint64_t numElements =
mlir::cast<cir::VectorType>(op.getVec().getType()).getSize();
uint64_t maskBits = llvm::NextPowerOf2(numElements - 1) - 1;
mlir::Value maskValue = mlir::LLVM::ConstantOp::create(
rewriter, loc, llvmIndexType,
rewriter.getIntegerAttr(llvmIndexType, maskBits));
mlir::Value maskVector =
mlir::LLVM::UndefOp::create(rewriter, loc, llvmIndexVecType);
for (uint64_t i = 0; i < numElements; ++i) {
mlir::Value idxValue =
mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);
maskVector = mlir::LLVM::InsertElementOp::create(rewriter, loc, maskVector,
maskValue, idxValue);
}
mlir::Value maskedIndices = mlir::LLVM::AndOp::create(
rewriter, loc, llvmIndexVecType, adaptor.getIndices(), maskVector);
mlir::Value result = mlir::LLVM::UndefOp::create(
rewriter, loc, getTypeConverter()->convertType(op.getVec().getType()));
for (uint64_t i = 0; i < numElements; ++i) {
mlir::Value iValue =
mlir::LLVM::ConstantOp::create(rewriter, loc, rewriter.getI64Type(), i);
mlir::Value indexValue = mlir::LLVM::ExtractElementOp::create(
rewriter, loc, maskedIndices, iValue);
mlir::Value valueAtIndex =
mlir::LLVM::ExtractElementOp::create(rewriter, loc, input, indexValue);
result = mlir::LLVM::InsertElementOp::create(rewriter, loc, result,
valueAtIndex, iValue);
}
rewriter.replaceOp(op, result);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVecTernaryOpLowering::matchAndRewrite(
cir::VecTernaryOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
// Convert `cond` into a vector of i1, then use that in a `select` op.
mlir::Value bitVec = mlir::LLVM::ICmpOp::create(
rewriter, op.getLoc(), mlir::LLVM::ICmpPredicate::ne, adaptor.getCond(),
mlir::LLVM::ZeroOp::create(
rewriter, op.getCond().getLoc(),
typeConverter->convertType(op.getCond().getType())));
rewriter.replaceOpWithNewOp<mlir::LLVM::SelectOp>(
op, bitVec, adaptor.getLhs(), adaptor.getRhs());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMComplexAddOpLowering::matchAndRewrite(
cir::ComplexAddOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Value lhs = adaptor.getLhs();
mlir::Value rhs = adaptor.getRhs();
mlir::Location loc = op.getLoc();
auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
mlir::Type complexElemTy =
getTypeConverter()->convertType(complexType.getElementType());
auto lhsReal = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));
auto lhsImag = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));
auto rhsReal = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));
auto rhsImag = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));
mlir::Value newReal;
mlir::Value newImag;
if (complexElemTy.isInteger()) {
newReal = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsReal,
rhsReal);
newImag = mlir::LLVM::AddOp::create(rewriter, loc, complexElemTy, lhsImag,
rhsImag);
} else {
assert(!cir::MissingFeatures::fastMathFlags());
assert(!cir::MissingFeatures::fpConstraints());
newReal = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsReal,
rhsReal);
newImag = mlir::LLVM::FAddOp::create(rewriter, loc, complexElemTy, lhsImag,
rhsImag);
}
mlir::Type complexLLVMTy =
getTypeConverter()->convertType(op.getResult().getType());
auto initialComplex =
mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy);
auto realComplex = mlir::LLVM::InsertValueOp::create(
rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0}));
rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
op, realComplex, newImag, ArrayRef(int64_t{1}));
return mlir::success();
}
mlir::LogicalResult CIRToLLVMComplexCreateOpLowering::matchAndRewrite(
cir::ComplexCreateOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type complexLLVMTy =
getTypeConverter()->convertType(op.getResult().getType());
auto initialComplex =
mlir::LLVM::UndefOp::create(rewriter, op->getLoc(), complexLLVMTy);
auto realComplex = mlir::LLVM::InsertValueOp::create(
rewriter, op->getLoc(), initialComplex, adaptor.getReal(),
ArrayRef(int64_t{0}));
auto complex = mlir::LLVM::InsertValueOp::create(
rewriter, op->getLoc(), realComplex, adaptor.getImag(),
ArrayRef(int64_t{1}));
rewriter.replaceOp(op, complex);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMComplexRealOpLowering::matchAndRewrite(
cir::ComplexRealOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
mlir::Value operand = adaptor.getOperand();
if (mlir::isa<cir::ComplexType>(op.getOperand().getType())) {
operand = mlir::LLVM::ExtractValueOp::create(
rewriter, op.getLoc(), resultLLVMTy, operand,
llvm::ArrayRef<std::int64_t>{0});
}
rewriter.replaceOp(op, operand);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMComplexSubOpLowering::matchAndRewrite(
cir::ComplexSubOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Value lhs = adaptor.getLhs();
mlir::Value rhs = adaptor.getRhs();
mlir::Location loc = op.getLoc();
auto complexType = mlir::cast<cir::ComplexType>(op.getLhs().getType());
mlir::Type complexElemTy =
getTypeConverter()->convertType(complexType.getElementType());
auto lhsReal = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{0}));
auto lhsImag = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, lhs, ArrayRef(int64_t{1}));
auto rhsReal = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{0}));
auto rhsImag = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, complexElemTy, rhs, ArrayRef(int64_t{1}));
mlir::Value newReal;
mlir::Value newImag;
if (complexElemTy.isInteger()) {
newReal = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsReal,
rhsReal);
newImag = mlir::LLVM::SubOp::create(rewriter, loc, complexElemTy, lhsImag,
rhsImag);
} else {
assert(!cir::MissingFeatures::fastMathFlags());
assert(!cir::MissingFeatures::fpConstraints());
newReal = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsReal,
rhsReal);
newImag = mlir::LLVM::FSubOp::create(rewriter, loc, complexElemTy, lhsImag,
rhsImag);
}
mlir::Type complexLLVMTy =
getTypeConverter()->convertType(op.getResult().getType());
auto initialComplex =
mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), complexLLVMTy);
auto realComplex = mlir::LLVM::InsertValueOp::create(
rewriter, op->getLoc(), initialComplex, newReal, ArrayRef(int64_t{0}));
rewriter.replaceOpWithNewOp<mlir::LLVM::InsertValueOp>(
op, realComplex, newImag, ArrayRef(int64_t{1}));
return mlir::success();
}
mlir::LogicalResult CIRToLLVMComplexImagOpLowering::matchAndRewrite(
cir::ComplexImagOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
mlir::Value operand = adaptor.getOperand();
mlir::Location loc = op.getLoc();
if (mlir::isa<cir::ComplexType>(op.getOperand().getType())) {
operand = mlir::LLVM::ExtractValueOp::create(
rewriter, loc, resultLLVMTy, operand, llvm::ArrayRef<std::int64_t>{1});
} else {
mlir::TypedAttr zeroAttr = rewriter.getZeroAttr(resultLLVMTy);
operand =
mlir::LLVM::ConstantOp::create(rewriter, loc, resultLLVMTy, zeroAttr);
}
rewriter.replaceOp(op, operand);
return mlir::success();
}
mlir::IntegerType computeBitfieldIntType(mlir::Type storageType,
mlir::MLIRContext *context,
unsigned &storageSize) {
return TypeSwitch<mlir::Type, mlir::IntegerType>(storageType)
.Case<cir::ArrayType>([&](cir::ArrayType atTy) {
storageSize = atTy.getSize() * 8;
return mlir::IntegerType::get(context, storageSize);
})
.Case<cir::IntType>([&](cir::IntType intTy) {
storageSize = intTy.getWidth();
return mlir::IntegerType::get(context, storageSize);
})
.Default([](mlir::Type) -> mlir::IntegerType {
llvm_unreachable(
"Either ArrayType or IntType expected for bitfields storage");
});
}
mlir::LogicalResult CIRToLLVMSetBitfieldOpLowering::matchAndRewrite(
cir::SetBitfieldOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPoint(op);
cir::BitfieldInfoAttr info = op.getBitfieldInfo();
uint64_t size = info.getSize();
uint64_t offset = info.getOffset();
mlir::Type storageType = info.getStorageType();
mlir::MLIRContext *context = storageType.getContext();
unsigned storageSize = 0;
mlir::IntegerType intType =
computeBitfieldIntType(storageType, context, storageSize);
mlir::Value srcVal = createIntCast(rewriter, adaptor.getSrc(), intType);
unsigned srcWidth = storageSize;
mlir::Value resultVal = srcVal;
if (storageSize != size) {
assert(storageSize > size && "Invalid bitfield size.");
mlir::Value val = mlir::LLVM::LoadOp::create(
rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),
op.getIsVolatile());
srcVal =
createAnd(rewriter, srcVal, llvm::APInt::getLowBitsSet(srcWidth, size));
resultVal = srcVal;
srcVal = createShL(rewriter, srcVal, offset);
// Mask out the original value.
val = createAnd(rewriter, val,
~llvm::APInt::getBitsSet(srcWidth, offset, offset + size));
// Or together the unchanged values and the source value.
srcVal = mlir::LLVM::OrOp::create(rewriter, op.getLoc(), val, srcVal);
}
mlir::LLVM::StoreOp::create(rewriter, op.getLoc(), srcVal, adaptor.getAddr(),
op.getAlignment(), op.getIsVolatile());
mlir::Type resultTy = getTypeConverter()->convertType(op.getType());
if (info.getIsSigned()) {
assert(size <= storageSize);
unsigned highBits = storageSize - size;
if (highBits) {
resultVal = createShL(rewriter, resultVal, highBits);
resultVal = createAShR(rewriter, resultVal, highBits);
}
}
resultVal = createIntCast(rewriter, resultVal,
mlir::cast<mlir::IntegerType>(resultTy),
info.getIsSigned());
rewriter.replaceOp(op, resultVal);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMComplexImagPtrOpLowering::matchAndRewrite(
cir::ComplexImagPtrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
cir::PointerType operandTy = op.getOperand().getType();
mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
mlir::Type elementLLVMTy =
getTypeConverter()->convertType(operandTy.getPointee());
mlir::LLVM::GEPArg gepIndices[2] = {{0}, {1}};
mlir::LLVM::GEPNoWrapFlags inboundsNuw =
mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,
inboundsNuw);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMComplexRealPtrOpLowering::matchAndRewrite(
cir::ComplexRealPtrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
cir::PointerType operandTy = op.getOperand().getType();
mlir::Type resultLLVMTy = getTypeConverter()->convertType(op.getType());
mlir::Type elementLLVMTy =
getTypeConverter()->convertType(operandTy.getPointee());
mlir::LLVM::GEPArg gepIndices[2] = {0, 0};
mlir::LLVM::GEPNoWrapFlags inboundsNuw =
mlir::LLVM::GEPNoWrapFlags::inbounds | mlir::LLVM::GEPNoWrapFlags::nuw;
rewriter.replaceOpWithNewOp<mlir::LLVM::GEPOp>(
op, resultLLVMTy, elementLLVMTy, adaptor.getOperand(), gepIndices,
inboundsNuw);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMGetBitfieldOpLowering::matchAndRewrite(
cir::GetBitfieldOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::OpBuilder::InsertionGuard guard(rewriter);
rewriter.setInsertionPoint(op);
cir::BitfieldInfoAttr info = op.getBitfieldInfo();
uint64_t size = info.getSize();
uint64_t offset = info.getOffset();
mlir::Type storageType = info.getStorageType();
mlir::MLIRContext *context = storageType.getContext();
unsigned storageSize = 0;
mlir::IntegerType intType =
computeBitfieldIntType(storageType, context, storageSize);
mlir::Value val = mlir::LLVM::LoadOp::create(
rewriter, op.getLoc(), intType, adaptor.getAddr(), op.getAlignment(),
op.getIsVolatile());
val = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), intType, val);
if (info.getIsSigned()) {
assert(static_cast<unsigned>(offset + size) <= storageSize);
unsigned highBits = storageSize - offset - size;
val = createShL(rewriter, val, highBits);
val = createAShR(rewriter, val, offset + highBits);
} else {
val = createLShR(rewriter, val, offset);
if (static_cast<unsigned>(offset) + size < storageSize)
val = createAnd(rewriter, val,
llvm::APInt::getLowBitsSet(storageSize, size));
}
mlir::Type resTy = getTypeConverter()->convertType(op.getType());
mlir::Value newOp = createIntCast(
rewriter, val, mlir::cast<mlir::IntegerType>(resTy), info.getIsSigned());
rewriter.replaceOp(op, newOp);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMIsConstantOpLowering::matchAndRewrite(
cir::IsConstantOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
rewriter.replaceOpWithNewOp<mlir::LLVM::IsConstantOp>(op, adaptor.getVal());
return mlir::success();
}
mlir::LogicalResult CIRToLLVMInlineAsmOpLowering::matchAndRewrite(
cir::InlineAsmOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type llResTy;
if (op.getNumResults())
llResTy = getTypeConverter()->convertType(op.getType(0));
cir::AsmFlavor dialect = op.getAsmFlavor();
mlir::LLVM::AsmDialect llDialect = dialect == cir::AsmFlavor::x86_att
? mlir::LLVM::AsmDialect::AD_ATT
: mlir::LLVM::AsmDialect::AD_Intel;
SmallVector<mlir::Attribute> opAttrs;
StringRef llvmAttrName = mlir::LLVM::InlineAsmOp::getElementTypeAttrName();
// this is for the lowering to LLVM from LLVM dialect. Otherwise, if we
// don't have the result (i.e. void type as a result of operation), the
// element type attribute will be attached to the whole instruction, but not
// to the operand
if (!op.getNumResults())
opAttrs.push_back(mlir::Attribute());
SmallVector<mlir::Value> llvmOperands;
SmallVector<mlir::Value> cirOperands;
for (auto const &[llvmOp, cirOp] :
zip(adaptor.getAsmOperands(), op.getAsmOperands())) {
append_range(llvmOperands, llvmOp);
append_range(cirOperands, cirOp);
}
// so far we infer the llvm dialect element type attr from
// CIR operand type.
for (auto const &[cirOpAttr, cirOp] :
zip(op.getOperandAttrs(), cirOperands)) {
if (!cirOpAttr) {
opAttrs.push_back(mlir::Attribute());
continue;
}
llvm::SmallVector<mlir::NamedAttribute, 1> attrs;
cir::PointerType typ = mlir::cast<cir::PointerType>(cirOp.getType());
mlir::TypeAttr typAttr = mlir::TypeAttr::get(convertTypeForMemory(
*getTypeConverter(), dataLayout, typ.getPointee()));
attrs.push_back(rewriter.getNamedAttr(llvmAttrName, typAttr));
mlir::DictionaryAttr newDict = rewriter.getDictionaryAttr(attrs);
opAttrs.push_back(newDict);
}
rewriter.replaceOpWithNewOp<mlir::LLVM::InlineAsmOp>(
op, llResTy, llvmOperands, op.getAsmStringAttr(), op.getConstraintsAttr(),
op.getSideEffectsAttr(),
/*is_align_stack*/ mlir::UnitAttr(),
/*tail_call_kind*/
mlir::LLVM::TailCallKindAttr::get(
getContext(), mlir::LLVM::tailcallkind::TailCallKind::None),
mlir::LLVM::AsmDialectAttr::get(getContext(), llDialect),
rewriter.getArrayAttr(opAttrs));
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVAStartOpLowering::matchAndRewrite(
cir::VAStartOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
adaptor.getArgList());
rewriter.replaceOpWithNewOp<mlir::LLVM::VaStartOp>(op, vaList);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVAEndOpLowering::matchAndRewrite(
cir::VAEndOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
adaptor.getArgList());
rewriter.replaceOpWithNewOp<mlir::LLVM::VaEndOp>(op, vaList);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVACopyOpLowering::matchAndRewrite(
cir::VACopyOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
auto dstList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
adaptor.getDstList());
auto srcList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
adaptor.getSrcList());
rewriter.replaceOpWithNewOp<mlir::LLVM::VaCopyOp>(op, dstList, srcList);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMVAArgOpLowering::matchAndRewrite(
cir::VAArgOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
assert(!cir::MissingFeatures::vaArgABILowering());
auto opaquePtr = mlir::LLVM::LLVMPointerType::get(getContext());
auto vaList = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(), opaquePtr,
adaptor.getArgList());
mlir::Type llvmType =
getTypeConverter()->convertType(op->getResultTypes().front());
if (!llvmType)
return mlir::failure();
rewriter.replaceOpWithNewOp<mlir::LLVM::VaArgOp>(op, llvmType, vaList);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMLabelOpLowering::matchAndRewrite(
cir::LabelOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::MLIRContext *ctx = rewriter.getContext();
mlir::Block *block = op->getBlock();
// A BlockTagOp cannot reside in the entry block. The address of the entry
// block cannot be taken
if (block->isEntryBlock()) {
mlir::Block *newBlock =
rewriter.splitBlock(op->getBlock(), mlir::Block::iterator(op));
rewriter.setInsertionPointToEnd(block);
mlir::LLVM::BrOp::create(rewriter, op.getLoc(), newBlock);
}
auto tagAttr =
mlir::LLVM::BlockTagAttr::get(ctx, blockInfoAddr.getTagIndex());
rewriter.setInsertionPoint(op);
auto blockTagOp =
mlir::LLVM::BlockTagOp::create(rewriter, op->getLoc(), tagAttr);
mlir::LLVM::LLVMFuncOp func = op->getParentOfType<mlir::LLVM::LLVMFuncOp>();
auto blockInfoAttr =
cir::BlockAddrInfoAttr::get(ctx, func.getSymName(), op.getLabel());
blockInfoAddr.mapBlockTag(blockInfoAttr, blockTagOp);
rewriter.eraseOp(op);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMBlockAddressOpLowering::matchAndRewrite(
cir::BlockAddressOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::MLIRContext *ctx = rewriter.getContext();
mlir::LLVM::BlockTagOp matchLabel =
blockInfoAddr.lookupBlockTag(op.getBlockAddrInfoAttr());
mlir::LLVM::BlockTagAttr tagAttr;
if (!matchLabel)
// If the BlockTagOp has not been emitted yet, use a placeholder.
// This will later be replaced with the correct tag index during
// `resolveBlockAddressOp`.
tagAttr = {};
else
tagAttr = matchLabel.getTag();
auto blkAddr = mlir::LLVM::BlockAddressAttr::get(
rewriter.getContext(), op.getBlockAddrInfoAttr().getFunc(), tagAttr);
rewriter.setInsertionPoint(op);
auto newOp = mlir::LLVM::BlockAddressOp::create(
rewriter, op.getLoc(), mlir::LLVM::LLVMPointerType::get(ctx), blkAddr);
if (!matchLabel)
blockInfoAddr.addUnresolvedBlockAddress(newOp, op.getBlockAddrInfoAttr());
rewriter.replaceOp(op, newOp);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMIndirectBrOpLowering::matchAndRewrite(
cir::IndirectBrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
llvm::SmallVector<mlir::Block *, 8> successors;
llvm::SmallVector<mlir::ValueRange, 8> succOperands;
bool poison = op.getPoison();
for (mlir::Block *succ : op->getSuccessors())
successors.push_back(succ);
for (mlir::ValueRange operand : op.getSuccOperands()) {
succOperands.push_back(operand);
}
auto llvmPtrType = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
mlir::Value targetAddr;
if (!poison) {
targetAddr = mlir::LLVM::BitcastOp::create(rewriter, op.getLoc(),
llvmPtrType, adaptor.getAddr());
} else {
targetAddr =
mlir::LLVM::PoisonOp::create(rewriter, op->getLoc(), llvmPtrType);
// Remove the block argument to avoid generating an empty PHI during
// lowering.
op->getBlock()->eraseArgument(0);
}
auto newOp = mlir::LLVM::IndirectBrOp::create(
rewriter, op.getLoc(), targetAddr, succOperands, successors);
rewriter.replaceOp(op, newOp);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMAwaitOpLowering::matchAndRewrite(
cir::AwaitOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
return mlir::failure();
}
mlir::LogicalResult CIRToLLVMCpuIdOpLowering::matchAndRewrite(
cir::CpuIdOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
mlir::Type i32Ty = rewriter.getI32Type();
mlir::Type i64Ty = rewriter.getI64Type();
mlir::Type i32PtrTy = mlir::LLVM::LLVMPointerType::get(i32Ty.getContext(), 0);
mlir::Type cpuidRetTy = mlir::LLVM::LLVMStructType::getLiteral(
rewriter.getContext(), {i32Ty, i32Ty, i32Ty, i32Ty});
mlir::Value functionId = adaptor.getFunctionId();
mlir::Value subFunctionId = adaptor.getSubFunctionId();
StringRef asmString, constraints;
mlir::ModuleOp moduleOp = op->getParentOfType<mlir::ModuleOp>();
llvm::Triple triple(
mlir::cast<mlir::StringAttr>(
moduleOp->getAttr(cir::CIRDialect::getTripleAttrName()))
.getValue());
if (triple.getArch() == llvm::Triple::x86) {
asmString = "cpuid";
constraints = "={ax},={bx},={cx},={dx},{ax},{cx}";
} else {
// x86-64 uses %rbx as the base register, so preserve it.
asmString = "xchgq %rbx, ${1:q}\n"
"cpuid\n"
"xchgq %rbx, ${1:q}";
constraints = "={ax},=r,={cx},={dx},0,2";
}
mlir::Value inlineAsm =
mlir::LLVM::InlineAsmOp::create(
rewriter, op.getLoc(), cpuidRetTy, {functionId, subFunctionId},
rewriter.getStringAttr(asmString),
rewriter.getStringAttr(constraints),
/*has_side_effects=*/mlir::UnitAttr{},
/*is_align_stack=*/mlir::UnitAttr{},
/*tail_call_kind=*/mlir::LLVM::TailCallKindAttr{},
/*asm_dialect=*/mlir::LLVM::AsmDialectAttr{},
/*operand_attrs=*/mlir::ArrayAttr{})
.getResult(0);
mlir::Value basePtr = adaptor.getCpuInfo();
mlir::DataLayout layout(op->getParentOfType<mlir::ModuleOp>());
unsigned alignment = layout.getTypeABIAlignment(i32Ty);
for (unsigned i = 0; i < 4; i++) {
mlir::Value extracted =
mlir::LLVM::ExtractValueOp::create(rewriter, op.getLoc(), inlineAsm, i)
.getResult();
mlir::Value index = mlir::LLVM::ConstantOp::create(
rewriter, op.getLoc(), i64Ty, rewriter.getI64IntegerAttr(i));
llvm::SmallVector<mlir::Value, 1> gepIndices = {index};
mlir::Value storePtr = mlir::LLVM::GEPOp::create(
rewriter, op.getLoc(), i32PtrTy, i32Ty, basePtr,
gepIndices, mlir::LLVM::GEPNoWrapFlags::none)
.getResult();
mlir::LLVM::StoreOp::create(rewriter, op.getLoc(), extracted, storePtr,
alignment);
}
rewriter.eraseOp(op);
return mlir::success();
}
mlir::LogicalResult CIRToLLVMMemChrOpLowering::matchAndRewrite(
cir::MemChrOp op, OpAdaptor adaptor,
mlir::ConversionPatternRewriter &rewriter) const {
auto llvmPtrTy = mlir::LLVM::LLVMPointerType::get(rewriter.getContext());
mlir::Type srcTy = getTypeConverter()->convertType(op.getSrc().getType());
mlir::Type patternTy =
getTypeConverter()->convertType(op.getPattern().getType());
mlir::Type lenTy = getTypeConverter()->convertType(op.getLen().getType());
auto fnTy =
mlir::LLVM::LLVMFunctionType::get(llvmPtrTy, {srcTy, patternTy, lenTy},
/*isVarArg=*/false);
llvm::StringRef fnName = "memchr";
createLLVMFuncOpIfNotExist(rewriter, op, fnName, fnTy);
rewriter.replaceOpWithNewOp<mlir::LLVM::CallOp>(
op, mlir::TypeRange{llvmPtrTy}, fnName,
mlir::ValueRange{adaptor.getSrc(), adaptor.getPattern(),
adaptor.getLen()});
return mlir::success();
}
std::unique_ptr<mlir::Pass> createConvertCIRToLLVMPass() {
return std::make_unique<ConvertCIRToLLVMPass>();
}
void populateCIRToLLVMPasses(mlir::OpPassManager &pm) {
mlir::populateCIRPreLoweringPasses(pm);
pm.addPass(mlir::omp::createMarkDeclareTargetPass());
pm.addPass(createConvertCIRToLLVMPass());
}
std::unique_ptr<llvm::Module>
lowerDirectlyFromCIRToLLVMIR(mlir::ModuleOp mlirModule, LLVMContext &llvmCtx,
StringRef mlirSaveTempsOutFile) {
llvm::TimeTraceScope scope("lower from CIR to LLVM directly");
mlir::MLIRContext *mlirCtx = mlirModule.getContext();
mlir::PassManager pm(mlirCtx);
populateCIRToLLVMPasses(pm);
(void)mlir::applyPassManagerCLOptions(pm);
if (mlir::failed(pm.run(mlirModule))) {
// FIXME: Handle any errors where they occurs and return a nullptr here.
report_fatal_error(
"The pass manager failed to lower CIR to LLVMIR dialect!");
}
if (!mlirSaveTempsOutFile.empty()) {
std::error_code ec;
llvm::raw_fd_ostream out(mlirSaveTempsOutFile, ec);
if (!ec)
mlirModule->print(out);
}
mlir::registerBuiltinDialectTranslation(*mlirCtx);
mlir::registerLLVMDialectTranslation(*mlirCtx);
mlir::registerOpenMPDialectTranslation(*mlirCtx);
mlir::registerCIRDialectTranslation(*mlirCtx);
llvm::TimeTraceScope translateScope("translateModuleToLLVMIR");
StringRef moduleName = mlirModule.getName().value_or("CIRToLLVMModule");
std::unique_ptr<llvm::Module> llvmModule =
mlir::translateModuleToLLVMIR(mlirModule, llvmCtx, moduleName);
if (!llvmModule) {
// FIXME: Handle any errors where they occurs and return a nullptr here.
report_fatal_error("Lowering from LLVMIR dialect to llvm IR failed!");
}
return llvmModule;
}
} // namespace direct
} // namespace cir
Reference in New Issue View Git Blame Copy Permalink