shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/mlir/lib/Dialect/LLVMIR/IR/LLVMDialect.cpp
Erich Keane 5283f46615
[CIR] Add tons of function infra, plus a handful of attributes (#179811) 
This patch puts together a lot more of the CIR infrastructure for
function attributes, plus adds a bunch of 'TODO' messages for areas that
have been skipped.

Along the way, we also implement 8 attributes in some way: -Convergent
gets a little more work, to make the `noconvergent` C attribute have an
effect

-optsize/minsize are implemented, sourced from the command line

-nobuiltin is a call-only attribute that tells not to replace the
individual call with a builtin. This is a touch confusing, since
no-builtins is an attribute that means "don't replace anything in the
body of this function with builtins (from this list)". The spelling
confusion is existing, and it seems that changing the names away from
LLVM would be confusing.

-save_reg_params & zero_call_used_regs are boht pretty simple registers

-temp-func-name just passes a string to LLVM, consistent with existing
implementation.

-default-func-attrs is a difficult one. It takes command line arguments
and passes them as LLVM-IR attributes directly on functions/calls. In
the dialect, we are capturing these in their own attribute to pass them
on correctly. However, this is one we cannot recover from LLVM-IR for
obvious reasons, so we instead choose to let the 'passthrough' mechanism
work for those.
2026-02-06 14:37:48 +00:00

4697 lines
183 KiB
C++
Raw Blame History

//===- LLVMDialect.cpp - LLVM IR Ops and Dialect registration -------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This file defines the types and operation details for the LLVM IR dialect in
// MLIR, and the LLVM IR dialect. It also registers the dialect.
//
//===----------------------------------------------------------------------===//
#include "mlir/Dialect/LLVMIR/LLVMDialect.h"
#include "mlir/Dialect/LLVMIR/LLVMAttrs.h"
#include "mlir/Dialect/LLVMIR/LLVMTypes.h"
#include "mlir/IR/Attributes.h"
#include "mlir/IR/Builders.h"
#include "mlir/IR/BuiltinOps.h"
#include "mlir/IR/BuiltinTypes.h"
#include "mlir/IR/DialectImplementation.h"
#include "mlir/IR/MLIRContext.h"
#include "mlir/IR/Matchers.h"
#include "mlir/Interfaces/FunctionImplementation.h"
#include "mlir/Transforms/InliningUtils.h"
#include "llvm/ADT/APFloat.h"
#include "llvm/ADT/TypeSwitch.h"
#include "llvm/IR/DataLayout.h"
#include "llvm/Support/Error.h"
#include "LLVMDialectBytecode.h"
#include <numeric>
#include <optional>
using namespace mlir;
using namespace mlir::LLVM;
using mlir::LLVM::cconv::getMaxEnumValForCConv;
using mlir::LLVM::linkage::getMaxEnumValForLinkage;
using mlir::LLVM::tailcallkind::getMaxEnumValForTailCallKind;
#include "mlir/Dialect/LLVMIR/LLVMOpsDialect.cpp.inc"
//===----------------------------------------------------------------------===//
// Attribute Helpers
//===----------------------------------------------------------------------===//
static constexpr const char kElemTypeAttrName[] = "elem_type";
static auto processFMFAttr(ArrayRef<NamedAttribute> attrs) {
SmallVector<NamedAttribute, 8> filteredAttrs(
llvm::make_filter_range(attrs, [&](NamedAttribute attr) {
if (attr.getName() == "fastmathFlags") {
auto defAttr =
FastmathFlagsAttr::get(attr.getValue().getContext(), {});
return defAttr != attr.getValue();
}
return true;
}));
return filteredAttrs;
}
/// Verifies `symbol`'s use in `op` to ensure the symbol is a valid and
/// fully defined llvm.func.
static LogicalResult verifySymbolAttrUse(FlatSymbolRefAttr symbol,
Operation *op,
SymbolTableCollection &symbolTable) {
StringRef name = symbol.getValue();
auto func =
symbolTable.lookupNearestSymbolFrom<LLVMFuncOp>(op, symbol.getAttr());
if (!func)
return op->emitOpError("'")
<< name << "' does not reference a valid LLVM function";
if (func.isExternal())
return op->emitOpError("'") << name << "' does not have a definition";
return success();
}
/// Returns a boolean type that has the same shape as `type`. It supports both
/// fixed size vectors as well as scalable vectors.
static Type getI1SameShape(Type type) {
Type i1Type = IntegerType::get(type.getContext(), 1);
if (LLVM::isCompatibleVectorType(type))
return LLVM::getVectorType(i1Type, LLVM::getVectorNumElements(type));
return i1Type;
}
// Parses one of the keywords provided in the list `keywords` and returns the
// position of the parsed keyword in the list. If none of the keywords from the
// list is parsed, returns -1.
static int parseOptionalKeywordAlternative(OpAsmParser &parser,
ArrayRef<StringRef> keywords) {
for (const auto &en : llvm::enumerate(keywords)) {
if (succeeded(parser.parseOptionalKeyword(en.value())))
return en.index();
}
return -1;
}
namespace {
template <typename Ty>
struct EnumTraits {};
#define REGISTER_ENUM_TYPE(Ty) \
template <> \
struct EnumTraits<Ty> { \
static StringRef stringify(Ty value) { return stringify##Ty(value); } \
static unsigned getMaxEnumVal() { return getMaxEnumValFor##Ty(); } \
}
REGISTER_ENUM_TYPE(Linkage);
REGISTER_ENUM_TYPE(UnnamedAddr);
REGISTER_ENUM_TYPE(CConv);
REGISTER_ENUM_TYPE(TailCallKind);
REGISTER_ENUM_TYPE(Visibility);
} // namespace
/// Parse an enum from the keyword, or default to the provided default value.
/// The return type is the enum type by default, unless overridden with the
/// second template argument.
template <typename EnumTy, typename RetTy = EnumTy>
static RetTy parseOptionalLLVMKeyword(OpAsmParser &parser,
EnumTy defaultValue) {
SmallVector<StringRef, 10> names;
for (unsigned i = 0, e = EnumTraits<EnumTy>::getMaxEnumVal(); i <= e; ++i)
names.push_back(EnumTraits<EnumTy>::stringify(static_cast<EnumTy>(i)));
int index = parseOptionalKeywordAlternative(parser, names);
if (index == -1)
return static_cast<RetTy>(defaultValue);
return static_cast<RetTy>(index);
}
static void printLLVMLinkage(OpAsmPrinter &p, Operation *, LinkageAttr val) {
p << stringifyLinkage(val.getLinkage());
}
static ParseResult parseLLVMLinkage(OpAsmParser &p, LinkageAttr &val) {
val = LinkageAttr::get(
p.getContext(),
parseOptionalLLVMKeyword<LLVM::Linkage>(p, LLVM::Linkage::External));
return success();
}
static ArrayAttr getLLVMAlignParamForCompressExpand(OpBuilder &builder,
bool isExpandLoad,
uint64_t alignment = 1) {
// From
// https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics
// https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics
//
// The pointer alignment defaults to 1.
if (alignment == 1) {
return nullptr;
}
auto emptyDictAttr = builder.getDictionaryAttr({});
auto alignmentAttr = builder.getI64IntegerAttr(alignment);
auto namedAttr =
builder.getNamedAttr(LLVMDialect::getAlignAttrName(), alignmentAttr);
SmallVector<mlir::NamedAttribute> attrs = {namedAttr};
auto alignDictAttr = builder.getDictionaryAttr(attrs);
// From
// https://llvm.org/docs/LangRef.html#llvm-masked-expandload-intrinsics
// https://llvm.org/docs/LangRef.html#llvm-masked-compressstore-intrinsics
//
// The align parameter attribute can be provided for [expandload]'s first
// argument. The align parameter attribute can be provided for
// [compressstore]'s second argument.
int pos = isExpandLoad ? 0 : 1;
return pos == 0 ? builder.getArrayAttr(
{alignDictAttr, emptyDictAttr, emptyDictAttr})
: builder.getArrayAttr(
{emptyDictAttr, alignDictAttr, emptyDictAttr});
}
//===----------------------------------------------------------------------===//
// Operand bundle helpers.
//===----------------------------------------------------------------------===//
static void printOneOpBundle(OpAsmPrinter &p, OperandRange operands,
TypeRange operandTypes, StringRef tag) {
p.printString(tag);
p << "(";
if (!operands.empty()) {
p.printOperands(operands);
p << " : ";
llvm::interleaveComma(operandTypes, p);
}
p << ")";
}
static void printOpBundles(OpAsmPrinter &p, Operation *op,
OperandRangeRange opBundleOperands,
TypeRangeRange opBundleOperandTypes,
std::optional<ArrayAttr> opBundleTags) {
if (opBundleOperands.empty())
return;
assert(opBundleTags && "expect operand bundle tags");
p << "[";
llvm::interleaveComma(
llvm::zip(opBundleOperands, opBundleOperandTypes, *opBundleTags), p,
[&p](auto bundle) {
auto bundleTag = cast<StringAttr>(std::get<2>(bundle)).getValue();
printOneOpBundle(p, std::get<0>(bundle), std::get<1>(bundle),
bundleTag);
});
p << "]";
}
static ParseResult parseOneOpBundle(
OpAsmParser &p,
SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> &opBundleOperands,
SmallVector<SmallVector<Type>> &opBundleOperandTypes,
SmallVector<Attribute> &opBundleTags) {
SMLoc currentParserLoc = p.getCurrentLocation();
SmallVector<OpAsmParser::UnresolvedOperand> operands;
SmallVector<Type> types;
std::string tag;
if (p.parseString(&tag))
return p.emitError(currentParserLoc, "expect operand bundle tag");
if (p.parseLParen())
return failure();
if (p.parseOptionalRParen()) {
if (p.parseOperandList(operands) || p.parseColon() ||
p.parseTypeList(types) || p.parseRParen())
return failure();
}
opBundleOperands.push_back(std::move(operands));
opBundleOperandTypes.push_back(std::move(types));
opBundleTags.push_back(StringAttr::get(p.getContext(), tag));
return success();
}
static std::optional<ParseResult> parseOpBundles(
OpAsmParser &p,
SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> &opBundleOperands,
SmallVector<SmallVector<Type>> &opBundleOperandTypes,
ArrayAttr &opBundleTags) {
if (p.parseOptionalLSquare())
return std::nullopt;
if (succeeded(p.parseOptionalRSquare()))
return success();
SmallVector<Attribute> opBundleTagAttrs;
auto bundleParser = [&] {
return parseOneOpBundle(p, opBundleOperands, opBundleOperandTypes,
opBundleTagAttrs);
};
if (p.parseCommaSeparatedList(bundleParser))
return failure();
if (p.parseRSquare())
return failure();
opBundleTags = ArrayAttr::get(p.getContext(), opBundleTagAttrs);
return success();
}
//===----------------------------------------------------------------------===//
// Printing, parsing, folding and builder for LLVM::CmpOp.
//===----------------------------------------------------------------------===//
void ICmpOp::print(OpAsmPrinter &p) {
p << " \"" << stringifyICmpPredicate(getPredicate()) << "\" " << getOperand(0)
<< ", " << getOperand(1);
p.printOptionalAttrDict((*this)->getAttrs(), {"predicate"});
p << " : " << getLhs().getType();
}
void FCmpOp::print(OpAsmPrinter &p) {
p << " \"" << stringifyFCmpPredicate(getPredicate()) << "\" " << getOperand(0)
<< ", " << getOperand(1);
p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()), {"predicate"});
p << " : " << getLhs().getType();
}
// <operation> ::= `llvm.icmp` string-literal ssa-use `,` ssa-use
// attribute-dict? `:` type
// <operation> ::= `llvm.fcmp` string-literal ssa-use `,` ssa-use
// attribute-dict? `:` type
template <typename CmpPredicateType>
static ParseResult parseCmpOp(OpAsmParser &parser, OperationState &result) {
StringAttr predicateAttr;
OpAsmParser::UnresolvedOperand lhs, rhs;
Type type;
SMLoc predicateLoc, trailingTypeLoc;
if (parser.getCurrentLocation(&predicateLoc) ||
parser.parseAttribute(predicateAttr, "predicate", result.attributes) ||
parser.parseOperand(lhs) || parser.parseComma() ||
parser.parseOperand(rhs) ||
parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type) ||
parser.resolveOperand(lhs, type, result.operands) ||
parser.resolveOperand(rhs, type, result.operands))
return failure();
// Replace the string attribute `predicate` with an integer attribute.
int64_t predicateValue = 0;
if (std::is_same<CmpPredicateType, ICmpPredicate>()) {
std::optional<ICmpPredicate> predicate =
symbolizeICmpPredicate(predicateAttr.getValue());
if (!predicate)
return parser.emitError(predicateLoc)
<< "'" << predicateAttr.getValue()
<< "' is an incorrect value of the 'predicate' attribute";
predicateValue = static_cast<int64_t>(*predicate);
} else {
std::optional<FCmpPredicate> predicate =
symbolizeFCmpPredicate(predicateAttr.getValue());
if (!predicate)
return parser.emitError(predicateLoc)
<< "'" << predicateAttr.getValue()
<< "' is an incorrect value of the 'predicate' attribute";
predicateValue = static_cast<int64_t>(*predicate);
}
result.attributes.set("predicate",
parser.getBuilder().getI64IntegerAttr(predicateValue));
// The result type is either i1 or a vector type <? x i1> if the inputs are
// vectors.
if (!isCompatibleType(type))
return parser.emitError(trailingTypeLoc,
"expected LLVM dialect-compatible type");
result.addTypes(getI1SameShape(type));
return success();
}
ParseResult ICmpOp::parse(OpAsmParser &parser, OperationState &result) {
return parseCmpOp<ICmpPredicate>(parser, result);
}
ParseResult FCmpOp::parse(OpAsmParser &parser, OperationState &result) {
return parseCmpOp<FCmpPredicate>(parser, result);
}
/// Returns a scalar or vector boolean attribute of the given type.
static Attribute getBoolAttribute(Type type, MLIRContext *ctx, bool value) {
auto boolAttr = BoolAttr::get(ctx, value);
ShapedType shapedType = dyn_cast<ShapedType>(type);
if (!shapedType)
return boolAttr;
return DenseElementsAttr::get(shapedType, boolAttr);
}
OpFoldResult ICmpOp::fold(FoldAdaptor adaptor) {
if (getPredicate() != ICmpPredicate::eq &&
getPredicate() != ICmpPredicate::ne)
return {};
// cmpi(eq/ne, x, x) -> true/false
if (getLhs() == getRhs())
return getBoolAttribute(getType(), getContext(),
getPredicate() == ICmpPredicate::eq);
// cmpi(eq/ne, alloca, null) -> false/true
if (getLhs().getDefiningOp<AllocaOp>() && getRhs().getDefiningOp<ZeroOp>())
return getBoolAttribute(getType(), getContext(),
getPredicate() == ICmpPredicate::ne);
// cmpi(eq/ne, null, alloca) -> cmpi(eq/ne, alloca, null)
if (getLhs().getDefiningOp<ZeroOp>() && getRhs().getDefiningOp<AllocaOp>()) {
Value lhs = getLhs();
Value rhs = getRhs();
getLhsMutable().assign(rhs);
getRhsMutable().assign(lhs);
return getResult();
}
return {};
}
//===----------------------------------------------------------------------===//
// Printing, parsing and verification for LLVM::AllocaOp.
//===----------------------------------------------------------------------===//
void AllocaOp::print(OpAsmPrinter &p) {
auto funcTy =
FunctionType::get(getContext(), {getArraySize().getType()}, {getType()});
if (getInalloca())
p << " inalloca";
p << ' ' << getArraySize() << " x " << getElemType();
if (getAlignment() && *getAlignment() != 0)
p.printOptionalAttrDict((*this)->getAttrs(),
{kElemTypeAttrName, getInallocaAttrName()});
else
p.printOptionalAttrDict(
(*this)->getAttrs(),
{getAlignmentAttrName(), kElemTypeAttrName, getInallocaAttrName()});
p << " : " << funcTy;
}
// <operation> ::= `llvm.alloca` `inalloca`? ssa-use `x` type
// attribute-dict? `:` type `,` type
ParseResult AllocaOp::parse(OpAsmParser &parser, OperationState &result) {
OpAsmParser::UnresolvedOperand arraySize;
Type type, elemType;
SMLoc trailingTypeLoc;
if (succeeded(parser.parseOptionalKeyword("inalloca")))
result.addAttribute(getInallocaAttrName(result.name),
UnitAttr::get(parser.getContext()));
if (parser.parseOperand(arraySize) || parser.parseKeyword("x") ||
parser.parseType(elemType) ||
parser.parseOptionalAttrDict(result.attributes) || parser.parseColon() ||
parser.getCurrentLocation(&trailingTypeLoc) || parser.parseType(type))
return failure();
std::optional<NamedAttribute> alignmentAttr =
result.attributes.getNamed("alignment");
if (alignmentAttr.has_value()) {
auto alignmentInt = llvm::dyn_cast<IntegerAttr>(alignmentAttr->getValue());
if (!alignmentInt)
return parser.emitError(parser.getNameLoc(),
"expected integer alignment");
if (alignmentInt.getValue().isZero())
result.attributes.erase("alignment");
}
// Extract the result type from the trailing function type.
auto funcType = llvm::dyn_cast<FunctionType>(type);
if (!funcType || funcType.getNumInputs() != 1 ||
funcType.getNumResults() != 1)
return parser.emitError(
trailingTypeLoc,
"expected trailing function type with one argument and one result");
if (parser.resolveOperand(arraySize, funcType.getInput(0), result.operands))
return failure();
Type resultType = funcType.getResult(0);
if (auto ptrResultType = llvm::dyn_cast<LLVMPointerType>(resultType))
result.addAttribute(kElemTypeAttrName, TypeAttr::get(elemType));
result.addTypes({funcType.getResult(0)});
return success();
}
LogicalResult AllocaOp::verify() {
// Only certain target extension types can be used in 'alloca'.
if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getElemType());
targetExtType && !targetExtType.supportsMemOps())
return emitOpError()
<< "this target extension type cannot be used in alloca";
return success();
}
//===----------------------------------------------------------------------===//
// LLVM::BrOp
//===----------------------------------------------------------------------===//
SuccessorOperands BrOp::getSuccessorOperands(unsigned index) {
assert(index == 0 && "invalid successor index");
return SuccessorOperands(getDestOperandsMutable());
}
//===----------------------------------------------------------------------===//
// LLVM::CondBrOp
//===----------------------------------------------------------------------===//
SuccessorOperands CondBrOp::getSuccessorOperands(unsigned index) {
assert(index < getNumSuccessors() && "invalid successor index");
return SuccessorOperands(index == 0 ? getTrueDestOperandsMutable()
: getFalseDestOperandsMutable());
}
void CondBrOp::build(OpBuilder &builder, OperationState &result,
Value condition, Block *trueDest, ValueRange trueOperands,
Block *falseDest, ValueRange falseOperands,
std::optional<std::pair<uint32_t, uint32_t>> weights) {
DenseI32ArrayAttr weightsAttr;
if (weights)
weightsAttr =
builder.getDenseI32ArrayAttr({static_cast<int32_t>(weights->first),
static_cast<int32_t>(weights->second)});
build(builder, result, condition, trueOperands, falseOperands, weightsAttr,
/*loop_annotation=*/{}, trueDest, falseDest);
}
//===----------------------------------------------------------------------===//
// LLVM::SwitchOp
//===----------------------------------------------------------------------===//
void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
Block *defaultDestination, ValueRange defaultOperands,
DenseIntElementsAttr caseValues,
BlockRange caseDestinations,
ArrayRef<ValueRange> caseOperands,
ArrayRef<int32_t> branchWeights) {
DenseI32ArrayAttr weightsAttr;
if (!branchWeights.empty())
weightsAttr = builder.getDenseI32ArrayAttr(branchWeights);
build(builder, result, value, defaultOperands, caseOperands, caseValues,
weightsAttr, defaultDestination, caseDestinations);
}
void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
Block *defaultDestination, ValueRange defaultOperands,
ArrayRef<APInt> caseValues, BlockRange caseDestinations,
ArrayRef<ValueRange> caseOperands,
ArrayRef<int32_t> branchWeights) {
DenseIntElementsAttr caseValuesAttr;
if (!caseValues.empty()) {
ShapedType caseValueType = VectorType::get(
static_cast<int64_t>(caseValues.size()), value.getType());
caseValuesAttr = DenseIntElementsAttr::get(caseValueType, caseValues);
}
build(builder, result, value, defaultDestination, defaultOperands,
caseValuesAttr, caseDestinations, caseOperands, branchWeights);
}
void SwitchOp::build(OpBuilder &builder, OperationState &result, Value value,
Block *defaultDestination, ValueRange defaultOperands,
ArrayRef<int32_t> caseValues, BlockRange caseDestinations,
ArrayRef<ValueRange> caseOperands,
ArrayRef<int32_t> branchWeights) {
DenseIntElementsAttr caseValuesAttr;
if (!caseValues.empty()) {
ShapedType caseValueType = VectorType::get(
static_cast<int64_t>(caseValues.size()), value.getType());
caseValuesAttr = DenseIntElementsAttr::get(caseValueType, caseValues);
}
build(builder, result, value, defaultDestination, defaultOperands,
caseValuesAttr, caseDestinations, caseOperands, branchWeights);
}
/// <cases> ::= `[` (case (`,` case )* )? `]`
/// <case> ::= integer `:` bb-id (`(` ssa-use-and-type-list `)`)?
static ParseResult parseSwitchOpCases(
OpAsmParser &parser, Type flagType, DenseIntElementsAttr &caseValues,
SmallVectorImpl<Block *> &caseDestinations,
SmallVectorImpl<SmallVector<OpAsmParser::UnresolvedOperand>> &caseOperands,
SmallVectorImpl<SmallVector<Type>> &caseOperandTypes) {
if (failed(parser.parseLSquare()))
return failure();
if (succeeded(parser.parseOptionalRSquare()))
return success();
SmallVector<APInt> values;
unsigned bitWidth = flagType.getIntOrFloatBitWidth();
auto parseCase = [&]() {
int64_t value = 0;
if (failed(parser.parseInteger(value)))
return failure();
values.push_back(APInt(bitWidth, value, /*isSigned=*/true));
Block *destination;
SmallVector<OpAsmParser::UnresolvedOperand> operands;
SmallVector<Type> operandTypes;
if (parser.parseColon() || parser.parseSuccessor(destination))
return failure();
if (!parser.parseOptionalLParen()) {
if (parser.parseOperandList(operands, OpAsmParser::Delimiter::None,
/*allowResultNumber=*/false) ||
parser.parseColonTypeList(operandTypes) || parser.parseRParen())
return failure();
}
caseDestinations.push_back(destination);
caseOperands.emplace_back(operands);
caseOperandTypes.emplace_back(operandTypes);
return success();
};
if (failed(parser.parseCommaSeparatedList(parseCase)))
return failure();
ShapedType caseValueType =
VectorType::get(static_cast<int64_t>(values.size()), flagType);
caseValues = DenseIntElementsAttr::get(caseValueType, values);
return parser.parseRSquare();
}
static void printSwitchOpCases(OpAsmPrinter &p, SwitchOp op, Type flagType,
DenseIntElementsAttr caseValues,
SuccessorRange caseDestinations,
OperandRangeRange caseOperands,
const TypeRangeRange &caseOperandTypes) {
p << '[';
p.printNewline();
if (!caseValues) {
p << ']';
return;
}
size_t index = 0;
llvm::interleave(
llvm::zip(caseValues, caseDestinations),
[&](auto i) {
p << " ";
p << std::get<0>(i);
p << ": ";
p.printSuccessorAndUseList(std::get<1>(i), caseOperands[index++]);
},
[&] {
p << ',';
p.printNewline();
});
p.printNewline();
p << ']';
}
LogicalResult SwitchOp::verify() {
if ((!getCaseValues() && !getCaseDestinations().empty()) ||
(getCaseValues() &&
getCaseValues()->size() !=
static_cast<int64_t>(getCaseDestinations().size())))
return emitOpError("expects number of case values to match number of "
"case destinations");
if (getCaseValues() &&
getValue().getType() != getCaseValues()->getElementType())
return emitError("expects case value type to match condition value type");
return success();
}
SuccessorOperands SwitchOp::getSuccessorOperands(unsigned index) {
assert(index < getNumSuccessors() && "invalid successor index");
return SuccessorOperands(index == 0 ? getDefaultOperandsMutable()
: getCaseOperandsMutable(index - 1));
}
//===----------------------------------------------------------------------===//
// Code for LLVM::GEPOp.
//===----------------------------------------------------------------------===//
GEPIndicesAdaptor<ValueRange> GEPOp::getIndices() {
return GEPIndicesAdaptor<ValueRange>(getRawConstantIndicesAttr(),
getDynamicIndices());
}
/// Returns the elemental type of any LLVM-compatible vector type or self.
static Type extractVectorElementType(Type type) {
if (auto vectorType = llvm::dyn_cast<VectorType>(type))
return vectorType.getElementType();
return type;
}
/// Destructures the 'indices' parameter into 'rawConstantIndices' and
/// 'dynamicIndices', encoding the former in the process. In the process,
/// dynamic indices which are used to index into a structure type are converted
/// to constant indices when possible. To do this, the GEPs element type should
/// be passed as first parameter.
static void destructureIndices(Type currType, ArrayRef<GEPArg> indices,
SmallVectorImpl<int32_t> &rawConstantIndices,
SmallVectorImpl<Value> &dynamicIndices) {
for (const GEPArg &iter : indices) {
// If the thing we are currently indexing into is a struct we must turn
// any integer constants into constant indices. If this is not possible
// we don't do anything here. The verifier will catch it and emit a proper
// error. All other canonicalization is done in the fold method.
bool requiresConst = !rawConstantIndices.empty() &&
isa_and_nonnull<LLVMStructType>(currType);
if (Value val = llvm::dyn_cast_if_present<Value>(iter)) {
APInt intC;
if (requiresConst && matchPattern(val, m_ConstantInt(&intC)) &&
intC.isSignedIntN(kGEPConstantBitWidth)) {
rawConstantIndices.push_back(intC.getSExtValue());
} else {
rawConstantIndices.push_back(GEPOp::kDynamicIndex);
dynamicIndices.push_back(val);
}
} else {
rawConstantIndices.push_back(cast<GEPConstantIndex>(iter));
}
// Skip for very first iteration of this loop. First index does not index
// within the aggregates, but is just a pointer offset.
if (rawConstantIndices.size() == 1 || !currType)
continue;
currType = TypeSwitch<Type, Type>(currType)
.Case<VectorType, LLVMArrayType>([](auto containerType) {
return containerType.getElementType();
})
.Case([&](LLVMStructType structType) -> Type {
int64_t memberIndex = rawConstantIndices.back();
if (memberIndex >= 0 && static_cast<size_t>(memberIndex) <
structType.getBody().size())
return structType.getBody()[memberIndex];
return nullptr;
})
.Default(nullptr);
}
}
void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
Type elementType, Value basePtr, ArrayRef<GEPArg> indices,
GEPNoWrapFlags noWrapFlags,
ArrayRef<NamedAttribute> attributes) {
SmallVector<int32_t> rawConstantIndices;
SmallVector<Value> dynamicIndices;
destructureIndices(elementType, indices, rawConstantIndices, dynamicIndices);
result.addTypes(resultType);
result.addAttributes(attributes);
result.getOrAddProperties<Properties>().rawConstantIndices =
builder.getDenseI32ArrayAttr(rawConstantIndices);
result.getOrAddProperties<Properties>().noWrapFlags = noWrapFlags;
result.getOrAddProperties<Properties>().elem_type =
TypeAttr::get(elementType);
result.addOperands(basePtr);
result.addOperands(dynamicIndices);
}
void GEPOp::build(OpBuilder &builder, OperationState &result, Type resultType,
Type elementType, Value basePtr, ValueRange indices,
GEPNoWrapFlags noWrapFlags,
ArrayRef<NamedAttribute> attributes) {
build(builder, result, resultType, elementType, basePtr,
SmallVector<GEPArg>(indices), noWrapFlags, attributes);
}
static ParseResult
parseGEPIndices(OpAsmParser &parser,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &indices,
DenseI32ArrayAttr &rawConstantIndices) {
SmallVector<int32_t> constantIndices;
auto idxParser = [&]() -> ParseResult {
int32_t constantIndex;
OptionalParseResult parsedInteger =
parser.parseOptionalInteger(constantIndex);
if (parsedInteger.has_value()) {
if (failed(parsedInteger.value()))
return failure();
constantIndices.push_back(constantIndex);
return success();
}
constantIndices.push_back(LLVM::GEPOp::kDynamicIndex);
return parser.parseOperand(indices.emplace_back());
};
if (parser.parseCommaSeparatedList(idxParser))
return failure();
rawConstantIndices =
DenseI32ArrayAttr::get(parser.getContext(), constantIndices);
return success();
}
static void printGEPIndices(OpAsmPrinter &printer, LLVM::GEPOp gepOp,
OperandRange indices,
DenseI32ArrayAttr rawConstantIndices) {
llvm::interleaveComma(
GEPIndicesAdaptor<OperandRange>(rawConstantIndices, indices), printer,
[&](PointerUnion<IntegerAttr, Value> cst) {
if (Value val = llvm::dyn_cast_if_present<Value>(cst))
printer.printOperand(val);
else
printer << cast<IntegerAttr>(cst).getInt();
});
}
/// For the given `indices`, check if they comply with `baseGEPType`,
/// especially check against LLVMStructTypes nested within.
static LogicalResult
verifyStructIndices(Type baseGEPType, unsigned indexPos,
GEPIndicesAdaptor<ValueRange> indices,
function_ref<InFlightDiagnostic()> emitOpError) {
if (indexPos >= indices.size())
// Stop searching
return success();
return TypeSwitch<Type, LogicalResult>(baseGEPType)
.Case([&](LLVMStructType structType) -> LogicalResult {
auto attr = dyn_cast<IntegerAttr>(indices[indexPos]);
if (!attr)
return emitOpError() << "expected index " << indexPos
<< " indexing a struct to be constant";
int32_t gepIndex = attr.getInt();
ArrayRef<Type> elementTypes = structType.getBody();
if (gepIndex < 0 ||
static_cast<size_t>(gepIndex) >= elementTypes.size())
return emitOpError() << "index " << indexPos
<< " indexing a struct is out of bounds";
// Instead of recursively going into every children types, we only
// dive into the one indexed by gepIndex.
return verifyStructIndices(elementTypes[gepIndex], indexPos + 1,
indices, emitOpError);
})
.Case<VectorType, LLVMArrayType>(
[&](auto containerType) -> LogicalResult {
return verifyStructIndices(containerType.getElementType(),
indexPos + 1, indices, emitOpError);
})
.Default([&](auto otherType) -> LogicalResult {
return emitOpError()
<< "type " << otherType << " cannot be indexed (index #"
<< indexPos << ")";
});
}
/// Driver function around `verifyStructIndices`.
static LogicalResult
verifyStructIndices(Type baseGEPType, GEPIndicesAdaptor<ValueRange> indices,
function_ref<InFlightDiagnostic()> emitOpError) {
return verifyStructIndices(baseGEPType, /*indexPos=*/1, indices, emitOpError);
}
LogicalResult LLVM::GEPOp::verify() {
if (static_cast<size_t>(
llvm::count(getRawConstantIndices(), kDynamicIndex)) !=
getDynamicIndices().size())
return emitOpError("expected as many dynamic indices as specified in '")
<< getRawConstantIndicesAttrName().getValue() << "'";
if (getNoWrapFlags() == GEPNoWrapFlags::inboundsFlag)
return emitOpError("'inbounds_flag' cannot be used directly.");
return verifyStructIndices(getElemType(), getIndices(),
[&] { return emitOpError(); });
}
//===----------------------------------------------------------------------===//
// LoadOp
//===----------------------------------------------------------------------===//
void LoadOp::getEffects(
SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
&effects) {
effects.emplace_back(MemoryEffects::Read::get(), &getAddrMutable());
// Volatile operations can have target-specific read-write effects on
// memory besides the one referred to by the pointer operand.
// Similarly, atomic operations that are monotonic or stricter cause
// synchronization that from a language point-of-view, are arbitrary
// read-writes into memory.
if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
getOrdering() != AtomicOrdering::unordered)) {
effects.emplace_back(MemoryEffects::Write::get());
effects.emplace_back(MemoryEffects::Read::get());
}
}
/// Returns true if the given type is supported by atomic operations. All
/// integer, float, and pointer types with a power-of-two bitsize and a minimal
/// size of 8 bits are supported.
bool LLVM::isTypeCompatibleWithAtomicOp(Type type,
const DataLayout &dataLayout) {
if (!isa<IntegerType, LLVMPointerType>(type))
if (!isCompatibleFloatingPointType(type))
return false;
llvm::TypeSize bitWidth = dataLayout.getTypeSizeInBits(type);
if (bitWidth.isScalable())
return false;
// Needs to be at least 8 bits and a power of two.
return bitWidth >= 8 && (bitWidth & (bitWidth - 1)) == 0;
}
/// Verifies the attributes and the type of atomic memory access operations.
template <typename OpTy>
static LogicalResult
verifyAtomicMemOp(OpTy memOp, Type valueType,
ArrayRef<AtomicOrdering> unsupportedOrderings) {
if (memOp.getOrdering() != AtomicOrdering::not_atomic) {
DataLayout dataLayout = DataLayout::closest(memOp);
if (!isTypeCompatibleWithAtomicOp(valueType, dataLayout))
return memOp.emitOpError("unsupported type ")
<< valueType << " for atomic access";
if (llvm::is_contained(unsupportedOrderings, memOp.getOrdering()))
return memOp.emitOpError("unsupported ordering '")
<< stringifyAtomicOrdering(memOp.getOrdering()) << "'";
if (!memOp.getAlignment())
return memOp.emitOpError("expected alignment for atomic access");
return success();
}
if (memOp.getSyncscope())
return memOp.emitOpError(
"expected syncscope to be null for non-atomic access");
return success();
}
LogicalResult LoadOp::verify() {
Type valueType = getResult().getType();
return verifyAtomicMemOp(*this, valueType,
{AtomicOrdering::release, AtomicOrdering::acq_rel});
}
void LoadOp::build(OpBuilder &builder, OperationState &state, Type type,
Value addr, unsigned alignment, bool isVolatile,
bool isNonTemporal, bool isInvariant, bool isInvariantGroup,
AtomicOrdering ordering, StringRef syncscope) {
build(builder, state, type, addr,
alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
isNonTemporal, isInvariant, isInvariantGroup, ordering,
syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
/*dereferenceable=*/nullptr,
/*access_groups=*/nullptr,
/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr,
/*tbaa=*/nullptr);
}
//===----------------------------------------------------------------------===//
// StoreOp
//===----------------------------------------------------------------------===//
void StoreOp::getEffects(
SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
&effects) {
effects.emplace_back(MemoryEffects::Write::get(), &getAddrMutable());
// Volatile operations can have target-specific read-write effects on
// memory besides the one referred to by the pointer operand.
// Similarly, atomic operations that are monotonic or stricter cause
// synchronization that from a language point-of-view, are arbitrary
// read-writes into memory.
if (getVolatile_() || (getOrdering() != AtomicOrdering::not_atomic &&
getOrdering() != AtomicOrdering::unordered)) {
effects.emplace_back(MemoryEffects::Write::get());
effects.emplace_back(MemoryEffects::Read::get());
}
}
LogicalResult StoreOp::verify() {
Type valueType = getValue().getType();
return verifyAtomicMemOp(*this, valueType,
{AtomicOrdering::acquire, AtomicOrdering::acq_rel});
}
void StoreOp::build(OpBuilder &builder, OperationState &state, Value value,
Value addr, unsigned alignment, bool isVolatile,
bool isNonTemporal, bool isInvariantGroup,
AtomicOrdering ordering, StringRef syncscope) {
build(builder, state, value, addr,
alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
isNonTemporal, isInvariantGroup, ordering,
syncscope.empty() ? nullptr : builder.getStringAttr(syncscope),
/*access_groups=*/nullptr,
/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}
//===----------------------------------------------------------------------===//
// CallOp
//===----------------------------------------------------------------------===//
/// Gets the MLIR Op-like result types of a LLVMFunctionType.
static SmallVector<Type, 1> getCallOpResultTypes(LLVMFunctionType calleeType) {
SmallVector<Type, 1> results;
Type resultType = calleeType.getReturnType();
if (!isa<LLVM::LLVMVoidType>(resultType))
results.push_back(resultType);
return results;
}
/// Gets the variadic callee type for a LLVMFunctionType.
static TypeAttr getCallOpVarCalleeType(LLVMFunctionType calleeType) {
return calleeType.isVarArg() ? TypeAttr::get(calleeType) : nullptr;
}
/// Constructs a LLVMFunctionType from MLIR `results` and `args`.
static LLVMFunctionType getLLVMFuncType(MLIRContext *context, TypeRange results,
ValueRange args) {
Type resultType;
if (results.empty())
resultType = LLVMVoidType::get(context);
else
resultType = results.front();
return LLVMFunctionType::get(resultType, llvm::to_vector(args.getTypes()),
/*isVarArg=*/false);
}
void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
StringRef callee, ValueRange args) {
build(builder, state, results, builder.getStringAttr(callee), args);
}
void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
StringAttr callee, ValueRange args) {
build(builder, state, results, SymbolRefAttr::get(callee), args);
}
void CallOp::build(OpBuilder &builder, OperationState &state, TypeRange results,
FlatSymbolRefAttr callee, ValueRange args) {
assert(callee && "expected non-null callee in direct call builder");
build(builder, state, results,
/*var_callee_type=*/nullptr, callee, args, /*fastmathFlags=*/nullptr,
/*CConv=*/nullptr, /*TailCallKind=*/nullptr,
/*memory_effects=*/nullptr,
/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
/*noreturn=*/nullptr, /*returns_twice=*/nullptr, /*hot=*/nullptr,
/*cold=*/nullptr, /*noduplicate=*/nullptr,
/*no_caller_saved_registers=*/nullptr, /*nocallback=*/nullptr,
/*modular_format=*/nullptr, /*nobuiltins=*/nullptr,
/*allocsize=*/nullptr, /*optsize=*/nullptr, /*minsize=*/nullptr,
/*nobuiltin=*/nullptr, /*save_reg_params=*/nullptr,
/*zero_call_used_regs=*/nullptr, /*trap_func_name=*/nullptr,
/*default_func_attrs=*/nullptr,
/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
/*no_inline=*/nullptr, /*always_inline=*/nullptr,
/*inline_hint=*/nullptr);
}
void CallOp::build(OpBuilder &builder, OperationState &state,
LLVMFunctionType calleeType, StringRef callee,
ValueRange args) {
build(builder, state, calleeType, builder.getStringAttr(callee), args);
}
void CallOp::build(OpBuilder &builder, OperationState &state,
LLVMFunctionType calleeType, StringAttr callee,
ValueRange args) {
build(builder, state, calleeType, SymbolRefAttr::get(callee), args);
}
void CallOp::build(OpBuilder &builder, OperationState &state,
LLVMFunctionType calleeType, FlatSymbolRefAttr callee,
ValueRange args) {
build(builder, state, getCallOpResultTypes(calleeType),
getCallOpVarCalleeType(calleeType), callee, args,
/*fastmathFlags=*/nullptr,
/*CConv=*/nullptr,
/*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
/*convergent=*/nullptr,
/*no_unwind=*/nullptr, /*will_return=*/nullptr,
/*noreturn=*/nullptr,
/*returns_twice=*/nullptr, /*hot=*/nullptr,
/*cold=*/nullptr, /*noduplicate=*/nullptr,
/*no_caller_saved_registers=*/nullptr, /*nocallback=*/nullptr,
/*modular_format=*/nullptr, /*nobuiltins=*/nullptr,
/*allocsize=*/nullptr, /*optsize=*/nullptr, /*minsize=*/nullptr,
/*nobuiltin=*/nullptr, /*save_reg_params=*/nullptr,
/*zero_call_used_regs=*/nullptr, /*trap_func_name=*/nullptr,
/*default_func_attrs=*/nullptr,
/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
/*access_groups=*/nullptr,
/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
/*no_inline=*/nullptr, /*always_inline=*/nullptr,
/*inline_hint=*/nullptr);
}
void CallOp::build(OpBuilder &builder, OperationState &state,
LLVMFunctionType calleeType, ValueRange args) {
build(builder, state, getCallOpResultTypes(calleeType),
getCallOpVarCalleeType(calleeType),
/*callee=*/nullptr, args,
/*fastmathFlags=*/nullptr,
/*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
/*noreturn=*/nullptr,
/*returns_twice=*/nullptr, /*hot=*/nullptr,
/*cold=*/nullptr, /*noduplicate=*/nullptr,
/*no_caller_saved_registers=*/nullptr, /*nocallback=*/nullptr,
/*modular_format=*/nullptr, /*nobuiltins=*/nullptr,
/*allocsize=*/nullptr, /*optsize=*/nullptr, /*minsize=*/nullptr,
/*nobuiltin=*/nullptr, /*save_reg_params=*/nullptr,
/*zero_call_used_regs=*/nullptr, /*trap_func_name=*/nullptr,
/*default_func_attrs=*/nullptr,
/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
/*no_inline=*/nullptr, /*always_inline=*/nullptr,
/*inline_hint=*/nullptr);
}
void CallOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
ValueRange args) {
auto calleeType = func.getFunctionType();
build(builder, state, getCallOpResultTypes(calleeType),
getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), args,
/*fastmathFlags=*/nullptr,
/*CConv=*/nullptr, /*TailCallKind=*/nullptr, /*memory_effects=*/nullptr,
/*convergent=*/nullptr, /*no_unwind=*/nullptr, /*will_return=*/nullptr,
/*noreturn=*/nullptr,
/*returns_twice=*/nullptr, /*hot=*/nullptr,
/*cold=*/nullptr, /*noduplicate=*/nullptr,
/*no_caller_saved_registers=*/nullptr, /*nocallback=*/nullptr,
/*modular_format=*/nullptr, /*nobuiltins=*/nullptr,
/*allocsize=*/nullptr, /*optsize=*/nullptr, /*minsize=*/nullptr,
/*nobuiltin=*/nullptr, /*save_reg_params=*/nullptr,
/*zero_call_used_regs=*/nullptr, /*trap_func_name=*/nullptr,
/*default_func_attrs=*/nullptr,
/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{},
/*access_groups=*/nullptr, /*alias_scopes=*/nullptr,
/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr,
/*noalias_scopes=*/nullptr, /*tbaa=*/nullptr,
/*no_inline=*/nullptr, /*always_inline=*/nullptr,
/*inline_hint=*/nullptr);
}
CallInterfaceCallable CallOp::getCallableForCallee() {
// Direct call.
if (FlatSymbolRefAttr calleeAttr = getCalleeAttr())
return calleeAttr;
// Indirect call, callee Value is the first operand.
return getOperand(0);
}
void CallOp::setCalleeFromCallable(CallInterfaceCallable callee) {
// Direct call.
if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) {
auto symRef = cast<SymbolRefAttr>(callee);
return setCalleeAttr(cast<FlatSymbolRefAttr>(symRef));
}
// Indirect call, callee Value is the first operand.
return setOperand(0, cast<Value>(callee));
}
Operation::operand_range CallOp::getArgOperands() {
return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
}
MutableOperandRange CallOp::getArgOperandsMutable() {
return MutableOperandRange(*this, getCallee().has_value() ? 0 : 1,
getCalleeOperands().size());
}
/// Verify that an inlinable callsite of a debug-info-bearing function in a
/// debug-info-bearing function has a debug location attached to it. This
/// mirrors an LLVM IR verifier.
static LogicalResult verifyCallOpDebugInfo(CallOp callOp, LLVMFuncOp callee) {
if (callee.isExternal())
return success();
auto parentFunc = callOp->getParentOfType<FunctionOpInterface>();
if (!parentFunc)
return success();
auto hasSubprogram = [](Operation *op) {
return op->getLoc()
->findInstanceOf<FusedLocWith<LLVM::DISubprogramAttr>>() !=
nullptr;
};
if (!hasSubprogram(parentFunc) || !hasSubprogram(callee))
return success();
bool containsLoc = !isa<UnknownLoc>(callOp->getLoc());
if (!containsLoc)
return callOp.emitError()
<< "inlinable function call in a function with a DISubprogram "
"location must have a debug location";
return success();
}
/// Verify that the parameter and return types of the variadic callee type match
/// the `callOp` argument and result types.
template <typename OpTy>
static LogicalResult verifyCallOpVarCalleeType(OpTy callOp) {
std::optional<LLVMFunctionType> varCalleeType = callOp.getVarCalleeType();
if (!varCalleeType)
return success();
// Verify the variadic callee type is a variadic function type.
if (!varCalleeType->isVarArg())
return callOp.emitOpError(
"expected var_callee_type to be a variadic function type");
// Verify the variadic callee type has at most as many parameters as the call
// has argument operands.
if (varCalleeType->getNumParams() > callOp.getArgOperands().size())
return callOp.emitOpError("expected var_callee_type to have at most ")
<< callOp.getArgOperands().size() << " parameters";
// Verify the variadic callee type matches the call argument types.
for (auto [paramType, operand] :
llvm::zip(varCalleeType->getParams(), callOp.getArgOperands()))
if (paramType != operand.getType())
return callOp.emitOpError()
<< "var_callee_type parameter type mismatch: " << paramType
<< " != " << operand.getType();
// Verify the variadic callee type matches the call result type.
if (!callOp.getNumResults()) {
if (!isa<LLVMVoidType>(varCalleeType->getReturnType()))
return callOp.emitOpError("expected var_callee_type to return void");
} else {
if (callOp.getResult().getType() != varCalleeType->getReturnType())
return callOp.emitOpError("var_callee_type return type mismatch: ")
<< varCalleeType->getReturnType()
<< " != " << callOp.getResult().getType();
}
return success();
}
template <typename OpType>
static LogicalResult verifyOperandBundles(OpType &op) {
OperandRangeRange opBundleOperands = op.getOpBundleOperands();
std::optional<ArrayAttr> opBundleTags = op.getOpBundleTags();
auto isStringAttr = [](Attribute tagAttr) {
return isa<StringAttr>(tagAttr);
};
if (opBundleTags && !llvm::all_of(*opBundleTags, isStringAttr))
return op.emitError("operand bundle tag must be a StringAttr");
size_t numOpBundles = opBundleOperands.size();
size_t numOpBundleTags = opBundleTags ? opBundleTags->size() : 0;
if (numOpBundles != numOpBundleTags)
return op.emitError("expected ")
<< numOpBundles << " operand bundle tags, but actually got "
<< numOpBundleTags;
return success();
}
LogicalResult CallOp::verify() { return verifyOperandBundles(*this); }
LogicalResult CallOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
if (failed(verifyCallOpVarCalleeType(*this)))
return failure();
// Type for the callee, we'll get it differently depending if it is a direct
// or indirect call.
Type fnType;
bool isIndirect = false;
// If this is an indirect call, the callee attribute is missing.
FlatSymbolRefAttr calleeName = getCalleeAttr();
if (!calleeName) {
isIndirect = true;
if (!getNumOperands())
return emitOpError(
"must have either a `callee` attribute or at least an operand");
auto ptrType = llvm::dyn_cast<LLVMPointerType>(getOperand(0).getType());
if (!ptrType)
return emitOpError("indirect call expects a pointer as callee: ")
<< getOperand(0).getType();
return success();
} else {
Operation *callee =
symbolTable.lookupNearestSymbolFrom(*this, calleeName.getAttr());
if (!callee)
return emitOpError()
<< "'" << calleeName.getValue()
<< "' does not reference a symbol in the current scope";
if (auto fn = dyn_cast<LLVMFuncOp>(callee)) {
if (failed(verifyCallOpDebugInfo(*this, fn)))
return failure();
fnType = fn.getFunctionType();
} else if (auto ifunc = dyn_cast<IFuncOp>(callee)) {
fnType = ifunc.getIFuncType();
} else {
return emitOpError()
<< "'" << calleeName.getValue()
<< "' does not reference a valid LLVM function or IFunc";
}
}
LLVMFunctionType funcType = llvm::dyn_cast<LLVMFunctionType>(fnType);
if (!funcType)
return emitOpError("callee does not have a functional type: ") << fnType;
if (funcType.isVarArg() && !getVarCalleeType())
return emitOpError() << "missing var_callee_type attribute for vararg call";
// Verify that the operand and result types match the callee.
if (!funcType.isVarArg() &&
funcType.getNumParams() != (getCalleeOperands().size() - isIndirect))
return emitOpError() << "incorrect number of operands ("
<< (getCalleeOperands().size() - isIndirect)
<< ") for callee (expecting: "
<< funcType.getNumParams() << ")";
if (funcType.getNumParams() > (getCalleeOperands().size() - isIndirect))
return emitOpError() << "incorrect number of operands ("
<< (getCalleeOperands().size() - isIndirect)
<< ") for varargs callee (expecting at least: "
<< funcType.getNumParams() << ")";
for (unsigned i = 0, e = funcType.getNumParams(); i != e; ++i)
if (getOperand(i + isIndirect).getType() != funcType.getParamType(i))
return emitOpError() << "operand type mismatch for operand " << i << ": "
<< getOperand(i + isIndirect).getType()
<< " != " << funcType.getParamType(i);
if (getNumResults() == 0 &&
!llvm::isa<LLVM::LLVMVoidType>(funcType.getReturnType()))
return emitOpError() << "expected function call to produce a value";
if (getNumResults() != 0 &&
llvm::isa<LLVM::LLVMVoidType>(funcType.getReturnType()))
return emitOpError()
<< "calling function with void result must not produce values";
if (getNumResults() > 1)
return emitOpError()
<< "expected LLVM function call to produce 0 or 1 result";
if (getNumResults() && getResult().getType() != funcType.getReturnType())
return emitOpError() << "result type mismatch: " << getResult().getType()
<< " != " << funcType.getReturnType();
return success();
}
void CallOp::print(OpAsmPrinter &p) {
auto callee = getCallee();
bool isDirect = callee.has_value();
p << ' ';
// Print calling convention.
if (getCConv() != LLVM::CConv::C)
p << stringifyCConv(getCConv()) << ' ';
if (getTailCallKind() != LLVM::TailCallKind::None)
p << tailcallkind::stringifyTailCallKind(getTailCallKind()) << ' ';
// Print the direct callee if present as a function attribute, or an indirect
// callee (first operand) otherwise.
if (isDirect)
p.printSymbolName(callee.value());
else
p << getOperand(0);
auto args = getCalleeOperands().drop_front(isDirect ? 0 : 1);
p << '(' << args << ')';
// Print the variadic callee type if the call is variadic.
if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
p << " vararg(" << *varCalleeType << ")";
if (!getOpBundleOperands().empty()) {
p << " ";
printOpBundles(p, *this, getOpBundleOperands(),
getOpBundleOperands().getTypes(), getOpBundleTags());
}
p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
{getCalleeAttrName(), getTailCallKindAttrName(),
getVarCalleeTypeAttrName(), getCConvAttrName(),
getOperandSegmentSizesAttrName(),
getOpBundleSizesAttrName(),
getOpBundleTagsAttrName(), getArgAttrsAttrName(),
getResAttrsAttrName()});
p << " : ";
if (!isDirect)
p << getOperand(0).getType() << ", ";
// Reconstruct the MLIR function type from operand and result types.
call_interface_impl::printFunctionSignature(
p, args.getTypes(), getArgAttrsAttr(),
/*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
}
/// Parses the type of a call operation and resolves the operands if the parsing
/// succeeds. Returns failure otherwise.
static ParseResult parseCallTypeAndResolveOperands(
OpAsmParser &parser, OperationState &result, bool isDirect,
ArrayRef<OpAsmParser::UnresolvedOperand> operands,
SmallVectorImpl<DictionaryAttr> &argAttrs,
SmallVectorImpl<DictionaryAttr> &resultAttrs) {
SMLoc trailingTypesLoc = parser.getCurrentLocation();
SmallVector<Type> types;
if (parser.parseColon())
return failure();
if (!isDirect) {
types.emplace_back();
if (parser.parseType(types.back()))
return failure();
if (parser.parseOptionalComma())
return parser.emitError(
trailingTypesLoc, "expected indirect call to have 2 trailing types");
}
SmallVector<Type> argTypes;
SmallVector<Type> resTypes;
if (call_interface_impl::parseFunctionSignature(parser, argTypes, argAttrs,
resTypes, resultAttrs)) {
if (isDirect)
return parser.emitError(trailingTypesLoc,
"expected direct call to have 1 trailing types");
return parser.emitError(trailingTypesLoc,
"expected trailing function type");
}
if (resTypes.size() > 1)
return parser.emitError(trailingTypesLoc,
"expected function with 0 or 1 result");
if (resTypes.size() == 1 && llvm::isa<LLVM::LLVMVoidType>(resTypes[0]))
return parser.emitError(trailingTypesLoc,
"expected a non-void result type");
// The head element of the types list matches the callee type for
// indirect calls, while the types list is emtpy for direct calls.
// Append the function input types to resolve the call operation
// operands.
llvm::append_range(types, argTypes);
if (parser.resolveOperands(operands, types, parser.getNameLoc(),
result.operands))
return failure();
if (!resTypes.empty())
result.addTypes(resTypes);
return success();
}
/// Parses an optional function pointer operand before the call argument list
/// for indirect calls, or stops parsing at the function identifier otherwise.
static ParseResult parseOptionalCallFuncPtr(
OpAsmParser &parser,
SmallVectorImpl<OpAsmParser::UnresolvedOperand> &operands) {
OpAsmParser::UnresolvedOperand funcPtrOperand;
OptionalParseResult parseResult = parser.parseOptionalOperand(funcPtrOperand);
if (parseResult.has_value()) {
if (failed(*parseResult))
return *parseResult;
operands.push_back(funcPtrOperand);
}
return success();
}
static ParseResult resolveOpBundleOperands(
OpAsmParser &parser, SMLoc loc, OperationState &state,
ArrayRef<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands,
ArrayRef<SmallVector<Type>> opBundleOperandTypes,
StringAttr opBundleSizesAttrName) {
unsigned opBundleIndex = 0;
for (const auto &[operands, types] :
llvm::zip_equal(opBundleOperands, opBundleOperandTypes)) {
if (operands.size() != types.size())
return parser.emitError(loc, "expected ")
<< operands.size()
<< " types for operand bundle operands for operand bundle #"
<< opBundleIndex << ", but actually got " << types.size();
if (parser.resolveOperands(operands, types, loc, state.operands))
return failure();
}
SmallVector<int32_t> opBundleSizes;
opBundleSizes.reserve(opBundleOperands.size());
for (const auto &operands : opBundleOperands)
opBundleSizes.push_back(operands.size());
state.addAttribute(
opBundleSizesAttrName,
DenseI32ArrayAttr::get(parser.getContext(), opBundleSizes));
return success();
}
// <operation> ::= `llvm.call` (cconv)? (tailcallkind)? (function-id | ssa-use)
// `(` ssa-use-list `)`
// ( `vararg(` var-callee-type `)` )?
// ( `[` op-bundles-list `]` )?
// attribute-dict? `:` (type `,`)? function-type
ParseResult CallOp::parse(OpAsmParser &parser, OperationState &result) {
SymbolRefAttr funcAttr;
TypeAttr varCalleeType;
SmallVector<OpAsmParser::UnresolvedOperand> operands;
SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands;
SmallVector<SmallVector<Type>> opBundleOperandTypes;
ArrayAttr opBundleTags;
// Default to C Calling Convention if no keyword is provided.
result.addAttribute(
getCConvAttrName(result.name),
CConvAttr::get(parser.getContext(),
parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
result.addAttribute(
getTailCallKindAttrName(result.name),
TailCallKindAttr::get(parser.getContext(),
parseOptionalLLVMKeyword<TailCallKind>(
parser, LLVM::TailCallKind::None)));
// Parse a function pointer for indirect calls.
if (parseOptionalCallFuncPtr(parser, operands))
return failure();
bool isDirect = operands.empty();
// Parse a function identifier for direct calls.
if (isDirect)
if (parser.parseAttribute(funcAttr, "callee", result.attributes))
return failure();
// Parse the function arguments.
if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren))
return failure();
bool isVarArg = parser.parseOptionalKeyword("vararg").succeeded();
if (isVarArg) {
StringAttr varCalleeTypeAttrName =
CallOp::getVarCalleeTypeAttrName(result.name);
if (parser.parseLParen().failed() ||
parser
.parseAttribute(varCalleeType, varCalleeTypeAttrName,
result.attributes)
.failed() ||
parser.parseRParen().failed())
return failure();
}
SMLoc opBundlesLoc = parser.getCurrentLocation();
if (std::optional<ParseResult> result = parseOpBundles(
parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
result && failed(*result))
return failure();
if (opBundleTags && !opBundleTags.empty())
result.addAttribute(CallOp::getOpBundleTagsAttrName(result.name).getValue(),
opBundleTags);
if (parser.parseOptionalAttrDict(result.attributes))
return failure();
// Parse the trailing type list and resolve the operands.
SmallVector<DictionaryAttr> argAttrs;
SmallVector<DictionaryAttr> resultAttrs;
if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands,
argAttrs, resultAttrs))
return failure();
call_interface_impl::addArgAndResultAttrs(
parser.getBuilder(), result, argAttrs, resultAttrs,
getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
opBundleOperandTypes,
getOpBundleSizesAttrName(result.name)))
return failure();
int32_t numOpBundleOperands = 0;
for (const auto &operands : opBundleOperands)
numOpBundleOperands += operands.size();
result.addAttribute(
CallOp::getOperandSegmentSizeAttr(),
parser.getBuilder().getDenseI32ArrayAttr(
{static_cast<int32_t>(operands.size()), numOpBundleOperands}));
return success();
}
LLVMFunctionType CallOp::getCalleeFunctionType() {
if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
return *varCalleeType;
return getLLVMFuncType(getContext(), getResultTypes(), getArgOperands());
}
///===---------------------------------------------------------------------===//
/// LLVM::InvokeOp
///===---------------------------------------------------------------------===//
void InvokeOp::build(OpBuilder &builder, OperationState &state, LLVMFuncOp func,
ValueRange ops, Block *normal, ValueRange normalOps,
Block *unwind, ValueRange unwindOps) {
auto calleeType = func.getFunctionType();
build(builder, state, getCallOpResultTypes(calleeType),
getCallOpVarCalleeType(calleeType), SymbolRefAttr::get(func), ops,
/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, normalOps, unwindOps,
nullptr, nullptr, {}, {}, normal, unwind);
}
void InvokeOp::build(OpBuilder &builder, OperationState &state, TypeRange tys,
FlatSymbolRefAttr callee, ValueRange ops, Block *normal,
ValueRange normalOps, Block *unwind,
ValueRange unwindOps) {
build(builder, state, tys,
/*var_callee_type=*/nullptr, callee, ops, /*arg_attrs=*/nullptr,
/*res_attrs=*/nullptr, normalOps, unwindOps, nullptr, nullptr, {}, {},
normal, unwind);
}
void InvokeOp::build(OpBuilder &builder, OperationState &state,
LLVMFunctionType calleeType, FlatSymbolRefAttr callee,
ValueRange ops, Block *normal, ValueRange normalOps,
Block *unwind, ValueRange unwindOps) {
build(builder, state, getCallOpResultTypes(calleeType),
getCallOpVarCalleeType(calleeType), callee, ops,
/*arg_attrs=*/nullptr, /*res_attrs=*/nullptr, normalOps, unwindOps,
nullptr, nullptr, {}, {}, normal, unwind);
}
SuccessorOperands InvokeOp::getSuccessorOperands(unsigned index) {
assert(index < getNumSuccessors() && "invalid successor index");
return SuccessorOperands(index == 0 ? getNormalDestOperandsMutable()
: getUnwindDestOperandsMutable());
}
CallInterfaceCallable InvokeOp::getCallableForCallee() {
// Direct call.
if (FlatSymbolRefAttr calleeAttr = getCalleeAttr())
return calleeAttr;
// Indirect call, callee Value is the first operand.
return getOperand(0);
}
void InvokeOp::setCalleeFromCallable(CallInterfaceCallable callee) {
// Direct call.
if (FlatSymbolRefAttr calleeAttr = getCalleeAttr()) {
auto symRef = cast<SymbolRefAttr>(callee);
return setCalleeAttr(cast<FlatSymbolRefAttr>(symRef));
}
// Indirect call, callee Value is the first operand.
return setOperand(0, cast<Value>(callee));
}
Operation::operand_range InvokeOp::getArgOperands() {
return getCalleeOperands().drop_front(getCallee().has_value() ? 0 : 1);
}
MutableOperandRange InvokeOp::getArgOperandsMutable() {
return MutableOperandRange(*this, getCallee().has_value() ? 0 : 1,
getCalleeOperands().size());
}
LogicalResult InvokeOp::verify() {
if (failed(verifyCallOpVarCalleeType(*this)))
return failure();
Block *unwindDest = getUnwindDest();
if (unwindDest->empty())
return emitError("must have at least one operation in unwind destination");
// In unwind destination, first operation must be LandingpadOp
if (!isa<LandingpadOp>(unwindDest->front()))
return emitError("first operation in unwind destination should be a "
"llvm.landingpad operation");
if (failed(verifyOperandBundles(*this)))
return failure();
return success();
}
void InvokeOp::print(OpAsmPrinter &p) {
auto callee = getCallee();
bool isDirect = callee.has_value();
p << ' ';
// Print calling convention.
if (getCConv() != LLVM::CConv::C)
p << stringifyCConv(getCConv()) << ' ';
// Either function name or pointer
if (isDirect)
p.printSymbolName(callee.value());
else
p << getOperand(0);
p << '(' << getCalleeOperands().drop_front(isDirect ? 0 : 1) << ')';
p << " to ";
p.printSuccessorAndUseList(getNormalDest(), getNormalDestOperands());
p << " unwind ";
p.printSuccessorAndUseList(getUnwindDest(), getUnwindDestOperands());
// Print the variadic callee type if the invoke is variadic.
if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
p << " vararg(" << *varCalleeType << ")";
if (!getOpBundleOperands().empty()) {
p << " ";
printOpBundles(p, *this, getOpBundleOperands(),
getOpBundleOperands().getTypes(), getOpBundleTags());
}
p.printOptionalAttrDict((*this)->getAttrs(),
{getCalleeAttrName(), getOperandSegmentSizeAttr(),
getCConvAttrName(), getVarCalleeTypeAttrName(),
getOpBundleSizesAttrName(),
getOpBundleTagsAttrName(), getArgAttrsAttrName(),
getResAttrsAttrName()});
p << " : ";
if (!isDirect)
p << getOperand(0).getType() << ", ";
call_interface_impl::printFunctionSignature(
p, getCalleeOperands().drop_front(isDirect ? 0 : 1).getTypes(),
getArgAttrsAttr(),
/*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
}
// <operation> ::= `llvm.invoke` (cconv)? (function-id | ssa-use)
// `(` ssa-use-list `)`
// `to` bb-id (`[` ssa-use-and-type-list `]`)?
// `unwind` bb-id (`[` ssa-use-and-type-list `]`)?
// ( `vararg(` var-callee-type `)` )?
// ( `[` op-bundles-list `]` )?
// attribute-dict? `:` (type `,`)?
// function-type-with-argument-attributes
ParseResult InvokeOp::parse(OpAsmParser &parser, OperationState &result) {
SmallVector<OpAsmParser::UnresolvedOperand, 8> operands;
SymbolRefAttr funcAttr;
TypeAttr varCalleeType;
SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands;
SmallVector<SmallVector<Type>> opBundleOperandTypes;
ArrayAttr opBundleTags;
Block *normalDest, *unwindDest;
SmallVector<Value, 4> normalOperands, unwindOperands;
Builder &builder = parser.getBuilder();
// Default to C Calling Convention if no keyword is provided.
result.addAttribute(
getCConvAttrName(result.name),
CConvAttr::get(parser.getContext(),
parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
// Parse a function pointer for indirect calls.
if (parseOptionalCallFuncPtr(parser, operands))
return failure();
bool isDirect = operands.empty();
// Parse a function identifier for direct calls.
if (isDirect && parser.parseAttribute(funcAttr, "callee", result.attributes))
return failure();
// Parse the function arguments.
if (parser.parseOperandList(operands, OpAsmParser::Delimiter::Paren) ||
parser.parseKeyword("to") ||
parser.parseSuccessorAndUseList(normalDest, normalOperands) ||
parser.parseKeyword("unwind") ||
parser.parseSuccessorAndUseList(unwindDest, unwindOperands))
return failure();
bool isVarArg = parser.parseOptionalKeyword("vararg").succeeded();
if (isVarArg) {
StringAttr varCalleeTypeAttrName =
InvokeOp::getVarCalleeTypeAttrName(result.name);
if (parser.parseLParen().failed() ||
parser
.parseAttribute(varCalleeType, varCalleeTypeAttrName,
result.attributes)
.failed() ||
parser.parseRParen().failed())
return failure();
}
SMLoc opBundlesLoc = parser.getCurrentLocation();
if (std::optional<ParseResult> result = parseOpBundles(
parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
result && failed(*result))
return failure();
if (opBundleTags && !opBundleTags.empty())
result.addAttribute(
InvokeOp::getOpBundleTagsAttrName(result.name).getValue(),
opBundleTags);
if (parser.parseOptionalAttrDict(result.attributes))
return failure();
// Parse the trailing type list and resolve the function operands.
SmallVector<DictionaryAttr> argAttrs;
SmallVector<DictionaryAttr> resultAttrs;
if (parseCallTypeAndResolveOperands(parser, result, isDirect, operands,
argAttrs, resultAttrs))
return failure();
call_interface_impl::addArgAndResultAttrs(
parser.getBuilder(), result, argAttrs, resultAttrs,
getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
opBundleOperandTypes,
getOpBundleSizesAttrName(result.name)))
return failure();
result.addSuccessors({normalDest, unwindDest});
result.addOperands(normalOperands);
result.addOperands(unwindOperands);
int32_t numOpBundleOperands = 0;
for (const auto &operands : opBundleOperands)
numOpBundleOperands += operands.size();
result.addAttribute(
InvokeOp::getOperandSegmentSizeAttr(),
builder.getDenseI32ArrayAttr({static_cast<int32_t>(operands.size()),
static_cast<int32_t>(normalOperands.size()),
static_cast<int32_t>(unwindOperands.size()),
numOpBundleOperands}));
return success();
}
LLVMFunctionType InvokeOp::getCalleeFunctionType() {
if (std::optional<LLVMFunctionType> varCalleeType = getVarCalleeType())
return *varCalleeType;
return getLLVMFuncType(getContext(), getResultTypes(), getArgOperands());
}
///===----------------------------------------------------------------------===//
/// Verifying/Printing/Parsing for LLVM::LandingpadOp.
///===----------------------------------------------------------------------===//
LogicalResult LandingpadOp::verify() {
Value value;
if (LLVMFuncOp func = (*this)->getParentOfType<LLVMFuncOp>()) {
if (!func.getPersonality())
return emitError(
"llvm.landingpad needs to be in a function with a personality");
}
// Consistency of llvm.landingpad result types is checked in
// LLVMFuncOp::verify().
if (!getCleanup() && getOperands().empty())
return emitError("landingpad instruction expects at least one clause or "
"cleanup attribute");
for (unsigned idx = 0, ie = getNumOperands(); idx < ie; idx++) {
value = getOperand(idx);
bool isFilter = llvm::isa<LLVMArrayType>(value.getType());
if (isFilter) {
// FIXME: Verify filter clauses when arrays are appropriately handled
} else {
// catch - global addresses only.
// Bitcast ops should have global addresses as their args.
if (auto bcOp = value.getDefiningOp<BitcastOp>()) {
if (auto addrOp = bcOp.getArg().getDefiningOp<AddressOfOp>())
continue;
return emitError("constant clauses expected").attachNote(bcOp.getLoc())
<< "global addresses expected as operand to "
"bitcast used in clauses for landingpad";
}
// ZeroOp and AddressOfOp allowed
if (value.getDefiningOp<ZeroOp>())
continue;
if (value.getDefiningOp<AddressOfOp>())
continue;
return emitError("clause #")
<< idx << " is not a known constant - null, addressof, bitcast";
}
}
return success();
}
void LandingpadOp::print(OpAsmPrinter &p) {
p << (getCleanup() ? " cleanup " : " ");
// Clauses
for (auto value : getOperands()) {
// Similar to llvm - if clause is an array type then it is filter
// clause else catch clause
bool isArrayTy = llvm::isa<LLVMArrayType>(value.getType());
p << '(' << (isArrayTy ? "filter " : "catch ") << value << " : "
<< value.getType() << ") ";
}
p.printOptionalAttrDict((*this)->getAttrs(), {"cleanup"});
p << ": " << getType();
}
// <operation> ::= `llvm.landingpad` `cleanup`?
// ((`catch` | `filter`) operand-type ssa-use)* attribute-dict?
ParseResult LandingpadOp::parse(OpAsmParser &parser, OperationState &result) {
// Check for cleanup
if (succeeded(parser.parseOptionalKeyword("cleanup")))
result.addAttribute("cleanup", parser.getBuilder().getUnitAttr());
// Parse clauses with types
while (succeeded(parser.parseOptionalLParen()) &&
(succeeded(parser.parseOptionalKeyword("filter")) ||
succeeded(parser.parseOptionalKeyword("catch")))) {
OpAsmParser::UnresolvedOperand operand;
Type ty;
if (parser.parseOperand(operand) || parser.parseColon() ||
parser.parseType(ty) ||
parser.resolveOperand(operand, ty, result.operands) ||
parser.parseRParen())
return failure();
}
Type type;
if (parser.parseColon() || parser.parseType(type))
return failure();
result.addTypes(type);
return success();
}
//===----------------------------------------------------------------------===//
// ExtractValueOp
//===----------------------------------------------------------------------===//
/// Extract the type at `position` in the LLVM IR aggregate type
/// `containerType`. Each element of `position` is an index into a nested
/// aggregate type. Return the resulting type or emit an error.
static Type getInsertExtractValueElementType(
function_ref<InFlightDiagnostic(StringRef)> emitError, Type containerType,
ArrayRef<int64_t> position) {
Type llvmType = containerType;
if (!isCompatibleType(containerType)) {
emitError("expected LLVM IR Dialect type, got ") << containerType;
return {};
}
// Infer the element type from the structure type: iteratively step inside the
// type by taking the element type, indexed by the position attribute for
// structures. Check the position index before accessing, it is supposed to
// be in bounds.
for (int64_t idx : position) {
if (auto arrayType = llvm::dyn_cast<LLVMArrayType>(llvmType)) {
if (idx < 0 || static_cast<unsigned>(idx) >= arrayType.getNumElements()) {
emitError("position out of bounds: ") << idx;
return {};
}
llvmType = arrayType.getElementType();
} else if (auto structType = llvm::dyn_cast<LLVMStructType>(llvmType)) {
if (idx < 0 ||
static_cast<unsigned>(idx) >= structType.getBody().size()) {
emitError("position out of bounds: ") << idx;
return {};
}
llvmType = structType.getBody()[idx];
} else {
emitError("expected LLVM IR structure/array type, got: ") << llvmType;
return {};
}
}
return llvmType;
}
/// Extract the type at `position` in the wrapped LLVM IR aggregate type
/// `containerType`.
static Type getInsertExtractValueElementType(Type llvmType,
ArrayRef<int64_t> position) {
for (int64_t idx : position) {
if (auto structType = llvm::dyn_cast<LLVMStructType>(llvmType))
llvmType = structType.getBody()[idx];
else
llvmType = llvm::cast<LLVMArrayType>(llvmType).getElementType();
}
return llvmType;
}
/// Extracts the element at the given index from an attribute. For
/// `ElementsAttr` and `ArrayAttr`, returns the element at the specified index.
/// For `ZeroAttr`, `UndefAttr`, and `PoisonAttr`, returns the attribute itself
/// unchanged. Returns `nullptr` if the attribute is not one of these types or
/// if the index is out of bounds.
static Attribute extractElementAt(Attribute attr, size_t index) {
if (auto elementsAttr = dyn_cast<ElementsAttr>(attr)) {
if (index < static_cast<size_t>(elementsAttr.getNumElements()))
return elementsAttr.getValues<Attribute>()[index];
return nullptr;
}
if (auto arrayAttr = dyn_cast<ArrayAttr>(attr)) {
if (index < arrayAttr.getValue().size())
return arrayAttr[index];
return nullptr;
}
if (isa<ZeroAttr, UndefAttr, PoisonAttr>(attr))
return attr;
return nullptr;
}
OpFoldResult LLVM::ExtractValueOp::fold(FoldAdaptor adaptor) {
if (auto extractValueOp = getContainer().getDefiningOp<ExtractValueOp>()) {
SmallVector<int64_t, 4> newPos(extractValueOp.getPosition());
newPos.append(getPosition().begin(), getPosition().end());
setPosition(newPos);
getContainerMutable().set(extractValueOp.getContainer());
return getResult();
}
Attribute containerAttr;
if (matchPattern(getContainer(), m_Constant(&containerAttr))) {
for (int64_t pos : getPosition()) {
containerAttr = extractElementAt(containerAttr, pos);
if (!containerAttr)
return nullptr;
}
return containerAttr;
}
Value container = getContainer();
ArrayRef<int64_t> extractPos = getPosition();
while (auto insertValueOp = container.getDefiningOp<InsertValueOp>()) {
ArrayRef<int64_t> insertPos = insertValueOp.getPosition();
auto extractPosSize = extractPos.size();
auto insertPosSize = insertPos.size();
// Case 1: Exact match of positions.
if (extractPos == insertPos)
return insertValueOp.getValue();
// Case 2: Insert position is a prefix of extract position. Continue
// traversal with the inserted value. Example:
// ```
// %0 = llvm.insertvalue %arg1, %undef[0] : !llvm.struct<(i32, i32, i32)>
// %1 = llvm.insertvalue %arg2, %0[1] : !llvm.struct<(i32, i32, i32)>
// %2 = llvm.insertvalue %arg3, %1[2] : !llvm.struct<(i32, i32, i32)>
// %3 = llvm.insertvalue %2, %foo[0]
// : !llvm.struct<(struct<(i32, i32, i32)>, i64)>
// %4 = llvm.extractvalue %3[0, 0]
// : !llvm.struct<(struct<(i32, i32, i32)>, i64)>
// ```
// In the above example, %4 is folded to %arg1.
if (extractPosSize > insertPosSize &&
extractPos.take_front(insertPosSize) == insertPos) {
container = insertValueOp.getValue();
extractPos = extractPos.drop_front(insertPosSize);
continue;
}
// Case 3: Try to continue the traversal with the container value.
// If extract position is a prefix of insert position, stop propagating back
// as it will miss dependencies. For instance, %3 should not fold to %f0 in
// the following example:
// ```
// %1 = llvm.insertvalue %f0, %0[0, 0] :
// !llvm.array<4 x !llvm.array<4 x f32>>
// %2 = llvm.insertvalue %arr, %1[0] :
// !llvm.array<4 x !llvm.array<4 x f32>>
// %3 = llvm.extractvalue %2[0, 0] : !llvm.array<4 x !llvm.array<4 x f32>>
// ```
if (insertPosSize > extractPosSize &&
extractPos == insertPos.take_front(extractPosSize))
break;
// If neither a prefix, nor the exact position, we can extract out of the
// value being inserted into. Moreover, we can try again if that operand
// is itself an insertvalue expression.
container = insertValueOp.getContainer();
}
// We failed to resolve past this container either because it is not an
// InsertValueOp, or it is an InsertValueOp that partially overlaps with the
// value being extracted. Update to read from this container instead.
if (container == getContainer())
return {};
setPosition(extractPos);
getContainerMutable().assign(container);
return getResult();
}
LogicalResult ExtractValueOp::verify() {
auto emitError = [this](StringRef msg) { return emitOpError(msg); };
Type valueType = getInsertExtractValueElementType(
emitError, getContainer().getType(), getPosition());
if (!valueType)
return failure();
if (getRes().getType() != valueType)
return emitOpError() << "Type mismatch: extracting from "
<< getContainer().getType() << " should produce "
<< valueType << " but this op returns "
<< getRes().getType();
return success();
}
void ExtractValueOp::build(OpBuilder &builder, OperationState &state,
Value container, ArrayRef<int64_t> position) {
build(builder, state,
getInsertExtractValueElementType(container.getType(), position),
container, builder.getAttr<DenseI64ArrayAttr>(position));
}
//===----------------------------------------------------------------------===//
// InsertValueOp
//===----------------------------------------------------------------------===//
namespace {
/// Update any ExtractValueOps using a given InsertValueOp to instead read from
/// the closest InsertValueOp in the chain leading up to the current op that
/// writes to the same member. This traversal could be done entirely in
/// ExtractValueOp::fold, but doing it here significantly speeds things up
/// because we can handle several ExtractValueOps with a single traversal.
/// For instance, in this example:
/// %i0 = llvm.insertvalue %v0, %undef[0]
/// %i1 = llvm.insertvalue %v1, %0[1]
/// ...
/// %i999 = llvm.insertvalue %v999, %998[999]
/// %e0 = llvm.extractvalue %i999[0]
/// %e1 = llvm.extractvalue %i999[1]
/// ...
/// %e999 = llvm.extractvalue %i999[999]
/// Individually running the folder on each extractvalue would require
/// traversing the insertvalue chain 1000 times, but running this pattern on the
/// InsertValueOp would allow us to achieve the same result with a single
/// traversal. The resulting IR after this pattern will then be:
/// %i0 = llvm.insertvalue %v0, %undef[0]
/// %i1 = llvm.insertvalue %v1, %0[1]
/// ...
/// %i999 = llvm.insertvalue %v999, %998[999]
/// %e0 = llvm.extractvalue %i0[0]
/// %e1 = llvm.extractvalue %i1[1]
/// ...
/// %e999 = llvm.extractvalue %i999[999]
struct ResolveExtractValueSource : public OpRewritePattern<InsertValueOp> {
using OpRewritePattern::OpRewritePattern;
LogicalResult matchAndRewrite(InsertValueOp insertOp,
PatternRewriter &rewriter) const override {
bool changed = false;
// Map each position in the top-level struct to the ExtractOps that read
// from it. For the example in the doc-comment above this map will be empty
// when we visit ops %i0 - %i998. For %i999, it will contain:
// 0 -> { %e0 }, 1 -> { %e1 }, ... 999-> { %e999 }
DenseMap<int64_t, SmallVector<ExtractValueOp, 4>> posToExtractOps;
auto insertBaseIdx = insertOp.getPosition()[0];
for (auto &use : insertOp->getUses()) {
if (auto extractOp = dyn_cast<ExtractValueOp>(use.getOwner())) {
auto baseIdx = extractOp.getPosition()[0];
// We can skip reads of the member that insertOp writes to since they
// will not be updated.
if (baseIdx == insertBaseIdx)
continue;
posToExtractOps[baseIdx].push_back(extractOp);
}
}
// Walk up the chain of insertions and try to resolve the remaining
// extractions that access the same member.
Value nextContainer = insertOp.getContainer();
while (!posToExtractOps.empty()) {
auto curInsert =
dyn_cast_or_null<InsertValueOp>(nextContainer.getDefiningOp());
if (!curInsert)
break;
nextContainer = curInsert.getContainer();
// Check if any extractions read the member written by this insertion.
auto curInsertBaseIdx = curInsert.getPosition()[0];
auto it = posToExtractOps.find(curInsertBaseIdx);
if (it == posToExtractOps.end())
continue;
// Update the ExtractOps to read from the current insertion.
for (auto &extractOp : it->second) {
rewriter.modifyOpInPlace(extractOp, [&] {
extractOp.getContainerMutable().assign(curInsert);
});
}
// The entry should never be empty if it exists, so if we are at this
// point, set changed to true.
assert(!it->second.empty());
changed |= true;
posToExtractOps.erase(it);
}
// There was no insertion along the chain that wrote the member accessed by
// these extracts. So we can update them to use the top of the chain.
for (auto &[baseIdx, extracts] : posToExtractOps) {
for (auto &extractOp : extracts) {
rewriter.modifyOpInPlace(extractOp, [&] {
extractOp.getContainerMutable().assign(nextContainer);
});
}
assert(!extracts.empty() && "Empty list in map");
changed = true;
}
return success(changed);
}
};
} // namespace
void InsertValueOp::getCanonicalizationPatterns(RewritePatternSet &patterns,
MLIRContext *context) {
patterns.add<ResolveExtractValueSource>(context);
}
/// Infer the value type from the container type and position.
static ParseResult
parseInsertExtractValueElementType(AsmParser &parser, Type &valueType,
Type containerType,
DenseI64ArrayAttr position) {
valueType = getInsertExtractValueElementType(
[&](StringRef msg) {
return parser.emitError(parser.getCurrentLocation(), msg);
},
containerType, position.asArrayRef());
return success(!!valueType);
}
/// Nothing to print for an inferred type.
static void printInsertExtractValueElementType(AsmPrinter &printer,
Operation *op, Type valueType,
Type containerType,
DenseI64ArrayAttr position) {}
LogicalResult InsertValueOp::verify() {
auto emitError = [this](StringRef msg) { return emitOpError(msg); };
Type valueType = getInsertExtractValueElementType(
emitError, getContainer().getType(), getPosition());
if (!valueType)
return failure();
if (getValue().getType() != valueType)
return emitOpError() << "Type mismatch: cannot insert "
<< getValue().getType() << " into "
<< getContainer().getType();
return success();
}
//===----------------------------------------------------------------------===//
// ReturnOp
//===----------------------------------------------------------------------===//
LogicalResult ReturnOp::verify() {
auto parent = (*this)->getParentOfType<LLVMFuncOp>();
if (!parent)
return success();
Type expectedType = parent.getFunctionType().getReturnType();
if (llvm::isa<LLVMVoidType>(expectedType)) {
if (!getArg())
return success();
InFlightDiagnostic diag = emitOpError("expected no operands");
diag.attachNote(parent->getLoc()) << "when returning from function";
return diag;
}
if (!getArg()) {
if (llvm::isa<LLVMVoidType>(expectedType))
return success();
InFlightDiagnostic diag = emitOpError("expected 1 operand");
diag.attachNote(parent->getLoc()) << "when returning from function";
return diag;
}
if (expectedType != getArg().getType()) {
InFlightDiagnostic diag = emitOpError("mismatching result types");
diag.attachNote(parent->getLoc()) << "when returning from function";
return diag;
}
return success();
}
//===----------------------------------------------------------------------===//
// LLVM::AddressOfOp.
//===----------------------------------------------------------------------===//
GlobalOp AddressOfOp::getGlobal(SymbolTableCollection &symbolTable) {
return dyn_cast_or_null<GlobalOp>(
symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
}
LLVMFuncOp AddressOfOp::getFunction(SymbolTableCollection &symbolTable) {
return dyn_cast_or_null<LLVMFuncOp>(
symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
}
AliasOp AddressOfOp::getAlias(SymbolTableCollection &symbolTable) {
return dyn_cast_or_null<AliasOp>(
symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
}
IFuncOp AddressOfOp::getIFunc(SymbolTableCollection &symbolTable) {
return dyn_cast_or_null<IFuncOp>(
symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr()));
}
LogicalResult
AddressOfOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
Operation *symbol =
symbolTable.lookupSymbolIn(parentLLVMModule(*this), getGlobalNameAttr());
auto global = dyn_cast_or_null<GlobalOp>(symbol);
auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
auto alias = dyn_cast_or_null<AliasOp>(symbol);
auto ifunc = dyn_cast_or_null<IFuncOp>(symbol);
if (!global && !function && !alias && !ifunc)
return emitOpError("must reference a global defined by 'llvm.mlir.global', "
"'llvm.mlir.alias' or 'llvm.func' or 'llvm.mlir.ifunc'");
LLVMPointerType type = getType();
if ((global && global.getAddrSpace() != type.getAddressSpace()) ||
(alias && alias.getAddrSpace() != type.getAddressSpace()))
return emitOpError("pointer address space must match address space of the "
"referenced global or alias");
return success();
}
// AddressOfOp constant-folds to the global symbol name.
OpFoldResult LLVM::AddressOfOp::fold(FoldAdaptor) {
return getGlobalNameAttr();
}
//===----------------------------------------------------------------------===//
// LLVM::DSOLocalEquivalentOp
//===----------------------------------------------------------------------===//
LLVMFuncOp
DSOLocalEquivalentOp::getFunction(SymbolTableCollection &symbolTable) {
return dyn_cast_or_null<LLVMFuncOp>(symbolTable.lookupSymbolIn(
parentLLVMModule(*this), getFunctionNameAttr()));
}
AliasOp DSOLocalEquivalentOp::getAlias(SymbolTableCollection &symbolTable) {
return dyn_cast_or_null<AliasOp>(symbolTable.lookupSymbolIn(
parentLLVMModule(*this), getFunctionNameAttr()));
}
LogicalResult
DSOLocalEquivalentOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
Operation *symbol = symbolTable.lookupSymbolIn(parentLLVMModule(*this),
getFunctionNameAttr());
auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
auto alias = dyn_cast_or_null<AliasOp>(symbol);
if (!function && !alias)
return emitOpError(
"must reference a global defined by 'llvm.func' or 'llvm.mlir.alias'");
if (alias) {
if (alias.getInitializer()
.walk([&](AddressOfOp addrOp) {
if (addrOp.getGlobal(symbolTable))
return WalkResult::interrupt();
return WalkResult::advance();
})
.wasInterrupted())
return emitOpError("must reference an alias to a function");
}
if ((function && function.getLinkage() == LLVM::Linkage::ExternWeak) ||
(alias && alias.getLinkage() == LLVM::Linkage::ExternWeak))
return emitOpError(
"target function with 'extern_weak' linkage not allowed");
return success();
}
/// Fold a dso_local_equivalent operation to a dedicated dso_local_equivalent
/// attribute.
OpFoldResult DSOLocalEquivalentOp::fold(FoldAdaptor) {
return DSOLocalEquivalentAttr::get(getContext(), getFunctionNameAttr());
}
//===----------------------------------------------------------------------===//
// Verifier for LLVM::ComdatOp.
//===----------------------------------------------------------------------===//
void ComdatOp::build(OpBuilder &builder, OperationState &result,
StringRef symName) {
result.addAttribute(getSymNameAttrName(result.name),
builder.getStringAttr(symName));
Region *body = result.addRegion();
body->emplaceBlock();
}
LogicalResult ComdatOp::verifyRegions() {
Region &body = getBody();
for (Operation &op : body.getOps())
if (!isa<ComdatSelectorOp>(op))
return op.emitError(
"only comdat selector symbols can appear in a comdat region");
return success();
}
//===----------------------------------------------------------------------===//
// Builder, printer and verifier for LLVM::GlobalOp.
//===----------------------------------------------------------------------===//
void GlobalOp::build(OpBuilder &builder, OperationState &result, Type type,
bool isConstant, Linkage linkage, StringRef name,
Attribute value, uint64_t alignment, unsigned addrSpace,
bool dsoLocal, bool threadLocal, SymbolRefAttr comdat,
ArrayRef<NamedAttribute> attrs,
ArrayRef<Attribute> dbgExprs) {
result.addAttribute(getSymNameAttrName(result.name),
builder.getStringAttr(name));
result.addAttribute(getGlobalTypeAttrName(result.name), TypeAttr::get(type));
if (isConstant)
result.addAttribute(getConstantAttrName(result.name),
builder.getUnitAttr());
if (value)
result.addAttribute(getValueAttrName(result.name), value);
if (dsoLocal)
result.addAttribute(getDsoLocalAttrName(result.name),
builder.getUnitAttr());
if (threadLocal)
result.addAttribute(getThreadLocal_AttrName(result.name),
builder.getUnitAttr());
if (comdat)
result.addAttribute(getComdatAttrName(result.name), comdat);
// Only add an alignment attribute if the "alignment" input
// is different from 0. The value must also be a power of two, but
// this is tested in GlobalOp::verify, not here.
if (alignment != 0)
result.addAttribute(getAlignmentAttrName(result.name),
builder.getI64IntegerAttr(alignment));
result.addAttribute(getLinkageAttrName(result.name),
LinkageAttr::get(builder.getContext(), linkage));
if (addrSpace != 0)
result.addAttribute(getAddrSpaceAttrName(result.name),
builder.getI32IntegerAttr(addrSpace));
result.attributes.append(attrs.begin(), attrs.end());
if (!dbgExprs.empty())
result.addAttribute(getDbgExprsAttrName(result.name),
ArrayAttr::get(builder.getContext(), dbgExprs));
result.addRegion();
}
template <typename OpType>
static void printCommonGlobalAndAlias(OpAsmPrinter &p, OpType op) {
p << ' ' << stringifyLinkage(op.getLinkage()) << ' ';
StringRef visibility = stringifyVisibility(op.getVisibility_());
if (!visibility.empty())
p << visibility << ' ';
if (op.getThreadLocal_())
p << "thread_local ";
if (auto unnamedAddr = op.getUnnamedAddr()) {
StringRef str = stringifyUnnamedAddr(*unnamedAddr);
if (!str.empty())
p << str << ' ';
}
}
void GlobalOp::print(OpAsmPrinter &p) {
printCommonGlobalAndAlias<GlobalOp>(p, *this);
if (getConstant())
p << "constant ";
p.printSymbolName(getSymName());
p << '(';
if (auto value = getValueOrNull())
p.printAttribute(value);
p << ')';
if (auto comdat = getComdat())
p << " comdat(" << *comdat << ')';
// Note that the alignment attribute is printed using the
// default syntax here, even though it is an inherent attribute
// (as defined in https://mlir.llvm.org/docs/LangRef/#attributes)
p.printOptionalAttrDict((*this)->getAttrs(),
{SymbolTable::getSymbolAttrName(),
getGlobalTypeAttrName(), getConstantAttrName(),
getValueAttrName(), getLinkageAttrName(),
getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
getVisibility_AttrName(), getComdatAttrName()});
// Print the trailing type unless it's a string global.
if (llvm::dyn_cast_or_null<StringAttr>(getValueOrNull()))
return;
p << " : " << getType();
Region &initializer = getInitializerRegion();
if (!initializer.empty()) {
p << ' ';
p.printRegion(initializer, /*printEntryBlockArgs=*/false);
}
}
static LogicalResult verifyComdat(Operation *op,
std::optional<SymbolRefAttr> attr) {
if (!attr)
return success();
auto *comdatSelector = SymbolTable::lookupNearestSymbolFrom(op, *attr);
if (!isa_and_nonnull<ComdatSelectorOp>(comdatSelector))
return op->emitError() << "expected comdat symbol";
return success();
}
static LogicalResult verifyBlockTags(LLVMFuncOp funcOp) {
llvm::DenseSet<BlockTagAttr> blockTags;
// Note that presence of `BlockTagOp`s currently can't prevent an unrecheable
// block to be removed by canonicalizer's region simplify pass, which needs to
// be dialect aware to allow extra constraints to be described.
WalkResult res = funcOp.walk([&](BlockTagOp blockTagOp) {
if (blockTags.contains(blockTagOp.getTag())) {
blockTagOp.emitError()
<< "duplicate block tag '" << blockTagOp.getTag().getId()
<< "' in the same function: ";
return WalkResult::interrupt();
}
blockTags.insert(blockTagOp.getTag());
return WalkResult::advance();
});
return failure(res.wasInterrupted());
}
/// Parse common attributes that might show up in the same order in both
/// GlobalOp and AliasOp.
template <typename OpType>
static ParseResult parseCommonGlobalAndAlias(OpAsmParser &parser,
OperationState &result) {
MLIRContext *ctx = parser.getContext();
// Parse optional linkage, default to External.
result.addAttribute(
OpType::getLinkageAttrName(result.name),
LLVM::LinkageAttr::get(ctx, parseOptionalLLVMKeyword<Linkage>(
parser, LLVM::Linkage::External)));
// Parse optional visibility, default to Default.
result.addAttribute(OpType::getVisibility_AttrName(result.name),
parser.getBuilder().getI64IntegerAttr(
parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
parser, LLVM::Visibility::Default)));
if (succeeded(parser.parseOptionalKeyword("thread_local")))
result.addAttribute(OpType::getThreadLocal_AttrName(result.name),
parser.getBuilder().getUnitAttr());
// Parse optional UnnamedAddr, default to None.
result.addAttribute(OpType::getUnnamedAddrAttrName(result.name),
parser.getBuilder().getI64IntegerAttr(
parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
parser, LLVM::UnnamedAddr::None)));
return success();
}
// operation ::= `llvm.mlir.global` linkage? visibility?
// (`unnamed_addr` | `local_unnamed_addr`)?
// `thread_local`? `constant`? `@` identifier
// `(` attribute? `)` (`comdat(` symbol-ref-id `)`)?
// attribute-list? (`:` type)? region?
//
// The type can be omitted for string attributes, in which case it will be
// inferred from the value of the string as [strlen(value) x i8].
ParseResult GlobalOp::parse(OpAsmParser &parser, OperationState &result) {
// Call into common parsing between GlobalOp and AliasOp.
if (parseCommonGlobalAndAlias<GlobalOp>(parser, result).failed())
return failure();
if (succeeded(parser.parseOptionalKeyword("constant")))
result.addAttribute(getConstantAttrName(result.name),
parser.getBuilder().getUnitAttr());
StringAttr name;
if (parser.parseSymbolName(name, getSymNameAttrName(result.name),
result.attributes) ||
parser.parseLParen())
return failure();
Attribute value;
if (parser.parseOptionalRParen()) {
if (parser.parseAttribute(value, getValueAttrName(result.name),
result.attributes) ||
parser.parseRParen())
return failure();
}
if (succeeded(parser.parseOptionalKeyword("comdat"))) {
SymbolRefAttr comdat;
if (parser.parseLParen() || parser.parseAttribute(comdat) ||
parser.parseRParen())
return failure();
result.addAttribute(getComdatAttrName(result.name), comdat);
}
SmallVector<Type, 1> types;
if (parser.parseOptionalAttrDict(result.attributes) ||
parser.parseOptionalColonTypeList(types))
return failure();
if (types.size() > 1)
return parser.emitError(parser.getNameLoc(), "expected zero or one type");
Region &initRegion = *result.addRegion();
if (types.empty()) {
if (auto strAttr = llvm::dyn_cast_or_null<StringAttr>(value)) {
MLIRContext *context = parser.getContext();
auto arrayType = LLVM::LLVMArrayType::get(IntegerType::get(context, 8),
strAttr.getValue().size());
types.push_back(arrayType);
} else {
return parser.emitError(parser.getNameLoc(),
"type can only be omitted for string globals");
}
} else {
OptionalParseResult parseResult =
parser.parseOptionalRegion(initRegion, /*arguments=*/{},
/*argTypes=*/{});
if (parseResult.has_value() && failed(*parseResult))
return failure();
}
result.addAttribute(getGlobalTypeAttrName(result.name),
TypeAttr::get(types[0]));
return success();
}
static bool isZeroAttribute(Attribute value) {
if (auto intValue = llvm::dyn_cast<IntegerAttr>(value))
return intValue.getValue().isZero();
if (auto fpValue = llvm::dyn_cast<FloatAttr>(value))
return fpValue.getValue().isZero();
if (auto splatValue = llvm::dyn_cast<SplatElementsAttr>(value))
return isZeroAttribute(splatValue.getSplatValue<Attribute>());
if (auto elementsValue = llvm::dyn_cast<ElementsAttr>(value))
return llvm::all_of(elementsValue.getValues<Attribute>(), isZeroAttribute);
if (auto arrayValue = llvm::dyn_cast<ArrayAttr>(value))
return llvm::all_of(arrayValue.getValue(), isZeroAttribute);
return false;
}
LogicalResult GlobalOp::verify() {
bool validType = isCompatibleOuterType(getType())
? !llvm::isa<LLVMVoidType, LLVMTokenType,
LLVMMetadataType, LLVMLabelType>(getType())
: llvm::isa<PointerElementTypeInterface>(getType());
if (!validType)
return emitOpError(
"expects type to be a valid element type for an LLVM global");
if ((*this)->getParentOp() && !satisfiesLLVMModule((*this)->getParentOp()))
return emitOpError("must appear at the module level");
if (auto strAttr = llvm::dyn_cast_or_null<StringAttr>(getValueOrNull())) {
auto type = llvm::dyn_cast<LLVMArrayType>(getType());
IntegerType elementType =
type ? llvm::dyn_cast<IntegerType>(type.getElementType()) : nullptr;
if (!elementType || elementType.getWidth() != 8 ||
type.getNumElements() != strAttr.getValue().size())
return emitOpError(
"requires an i8 array type of the length equal to that of the string "
"attribute");
}
if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType())) {
if (!targetExtType.hasProperty(LLVMTargetExtType::CanBeGlobal))
return emitOpError()
<< "this target extension type cannot be used in a global";
if (Attribute value = getValueOrNull())
return emitOpError() << "global with target extension type can only be "
"initialized with zero-initializer";
}
if (getLinkage() == Linkage::Common) {
if (Attribute value = getValueOrNull()) {
if (!isZeroAttribute(value)) {
return emitOpError()
<< "expected zero value for '"
<< stringifyLinkage(Linkage::Common) << "' linkage";
}
}
}
if (getLinkage() == Linkage::Appending) {
if (!llvm::isa<LLVMArrayType>(getType())) {
return emitOpError() << "expected array type for '"
<< stringifyLinkage(Linkage::Appending)
<< "' linkage";
}
}
if (failed(verifyComdat(*this, getComdat())))
return failure();
std::optional<uint64_t> alignAttr = getAlignment();
if (alignAttr.has_value()) {
uint64_t value = alignAttr.value();
if (!llvm::isPowerOf2_64(value))
return emitError() << "alignment attribute is not a power of 2";
}
return success();
}
LogicalResult GlobalOp::verifyRegions() {
if (Block *b = getInitializerBlock()) {
ReturnOp ret = cast<ReturnOp>(b->getTerminator());
if (ret.operand_type_begin() == ret.operand_type_end())
return emitOpError("initializer region cannot return void");
if (*ret.operand_type_begin() != getType())
return emitOpError("initializer region type ")
<< *ret.operand_type_begin() << " does not match global type "
<< getType();
for (Operation &op : *b) {
auto iface = dyn_cast<MemoryEffectOpInterface>(op);
if (!iface || !iface.hasNoEffect())
return op.emitError()
<< "ops with side effects not allowed in global initializers";
}
if (getValueOrNull())
return emitOpError("cannot have both initializer value and region");
}
return success();
}
//===----------------------------------------------------------------------===//
// LLVM::GlobalCtorsOp
//===----------------------------------------------------------------------===//
static LogicalResult checkGlobalXtorData(Operation *op, ArrayAttr data) {
if (data.empty())
return success();
if (llvm::all_of(data.getAsRange<Attribute>(), [](Attribute v) {
return isa<FlatSymbolRefAttr, ZeroAttr>(v);
}))
return success();
return op->emitError("data element must be symbol or #llvm.zero");
}
LogicalResult
GlobalCtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
for (Attribute ctor : getCtors()) {
if (failed(verifySymbolAttrUse(llvm::cast<FlatSymbolRefAttr>(ctor), *this,
symbolTable)))
return failure();
}
return success();
}
LogicalResult GlobalCtorsOp::verify() {
if (checkGlobalXtorData(*this, getData()).failed())
return failure();
if (getCtors().size() == getPriorities().size() &&
getCtors().size() == getData().size())
return success();
return emitError(
"ctors, priorities, and data must have the same number of elements");
}
//===----------------------------------------------------------------------===//
// LLVM::GlobalDtorsOp
//===----------------------------------------------------------------------===//
LogicalResult
GlobalDtorsOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
for (Attribute dtor : getDtors()) {
if (failed(verifySymbolAttrUse(llvm::cast<FlatSymbolRefAttr>(dtor), *this,
symbolTable)))
return failure();
}
return success();
}
LogicalResult GlobalDtorsOp::verify() {
if (checkGlobalXtorData(*this, getData()).failed())
return failure();
if (getDtors().size() == getPriorities().size() &&
getDtors().size() == getData().size())
return success();
return emitError(
"dtors, priorities, and data must have the same number of elements");
}
//===----------------------------------------------------------------------===//
// Builder, printer and verifier for LLVM::AliasOp.
//===----------------------------------------------------------------------===//
void AliasOp::build(OpBuilder &builder, OperationState &result, Type type,
Linkage linkage, StringRef name, bool dsoLocal,
bool threadLocal, ArrayRef<NamedAttribute> attrs) {
result.addAttribute(getSymNameAttrName(result.name),
builder.getStringAttr(name));
result.addAttribute(getAliasTypeAttrName(result.name), TypeAttr::get(type));
if (dsoLocal)
result.addAttribute(getDsoLocalAttrName(result.name),
builder.getUnitAttr());
if (threadLocal)
result.addAttribute(getThreadLocal_AttrName(result.name),
builder.getUnitAttr());
result.addAttribute(getLinkageAttrName(result.name),
LinkageAttr::get(builder.getContext(), linkage));
result.attributes.append(attrs.begin(), attrs.end());
result.addRegion();
}
void AliasOp::print(OpAsmPrinter &p) {
printCommonGlobalAndAlias<AliasOp>(p, *this);
p.printSymbolName(getSymName());
p.printOptionalAttrDict((*this)->getAttrs(),
{SymbolTable::getSymbolAttrName(),
getAliasTypeAttrName(), getLinkageAttrName(),
getUnnamedAddrAttrName(), getThreadLocal_AttrName(),
getVisibility_AttrName()});
// Print the trailing type.
p << " : " << getType() << ' ';
// Print the initializer region.
p.printRegion(getInitializerRegion(), /*printEntryBlockArgs=*/false);
}
// operation ::= `llvm.mlir.alias` linkage? visibility?
// (`unnamed_addr` | `local_unnamed_addr`)?
// `thread_local`? `@` identifier
// `(` attribute? `)`
// attribute-list? `:` type region
//
ParseResult AliasOp::parse(OpAsmParser &parser, OperationState &result) {
// Call into common parsing between GlobalOp and AliasOp.
if (parseCommonGlobalAndAlias<AliasOp>(parser, result).failed())
return failure();
StringAttr name;
if (parser.parseSymbolName(name, getSymNameAttrName(result.name),
result.attributes))
return failure();
SmallVector<Type, 1> types;
if (parser.parseOptionalAttrDict(result.attributes) ||
parser.parseOptionalColonTypeList(types))
return failure();
if (types.size() > 1)
return parser.emitError(parser.getNameLoc(), "expected zero or one type");
Region &initRegion = *result.addRegion();
if (parser.parseRegion(initRegion).failed())
return failure();
result.addAttribute(getAliasTypeAttrName(result.name),
TypeAttr::get(types[0]));
return success();
}
LogicalResult AliasOp::verify() {
bool validType = isCompatibleOuterType(getType())
? !llvm::isa<LLVMVoidType, LLVMTokenType,
LLVMMetadataType, LLVMLabelType>(getType())
: llvm::isa<PointerElementTypeInterface>(getType());
if (!validType)
return emitOpError(
"expects type to be a valid element type for an LLVM global alias");
// This matches LLVM IR verification logic, see llvm/lib/IR/Verifier.cpp
switch (getLinkage()) {
case Linkage::External:
case Linkage::Internal:
case Linkage::Private:
case Linkage::Weak:
case Linkage::WeakODR:
case Linkage::Linkonce:
case Linkage::LinkonceODR:
case Linkage::AvailableExternally:
break;
default:
return emitOpError()
<< "'" << stringifyLinkage(getLinkage())
<< "' linkage not supported in aliases, available options: private, "
"internal, linkonce, weak, linkonce_odr, weak_odr, external or "
"available_externally";
}
return success();
}
LogicalResult AliasOp::verifyRegions() {
Block &b = getInitializerBlock();
auto ret = cast<ReturnOp>(b.getTerminator());
if (ret.getNumOperands() == 0 ||
!isa<LLVM::LLVMPointerType>(ret.getOperand(0).getType()))
return emitOpError("initializer region must always return a pointer");
for (Operation &op : b) {
auto iface = dyn_cast<MemoryEffectOpInterface>(op);
if (!iface || !iface.hasNoEffect())
return op.emitError()
<< "ops with side effects are not allowed in alias initializers";
}
return success();
}
unsigned AliasOp::getAddrSpace() {
Block &initializer = getInitializerBlock();
auto ret = cast<ReturnOp>(initializer.getTerminator());
auto ptrTy = cast<LLVMPointerType>(ret.getOperand(0).getType());
return ptrTy.getAddressSpace();
}
//===----------------------------------------------------------------------===//
// IFuncOp
//===----------------------------------------------------------------------===//
void IFuncOp::build(OpBuilder &builder, OperationState &result, StringRef name,
Type iFuncType, StringRef resolverName, Type resolverType,
Linkage linkage, LLVM::Visibility visibility) {
return build(builder, result, name, iFuncType, resolverName, resolverType,
linkage, /*dso_local=*/false, /*address_space=*/0,
UnnamedAddr::None, visibility);
}
LogicalResult IFuncOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
Operation *symbol =
symbolTable.lookupSymbolIn(parentLLVMModule(*this), getResolverAttr());
// This matches LLVM IR verification logic, see llvm/lib/IR/Verifier.cpp
auto resolver = dyn_cast<LLVMFuncOp>(symbol);
auto alias = dyn_cast<AliasOp>(symbol);
while (alias) {
Block &initBlock = alias.getInitializerBlock();
auto returnOp = cast<ReturnOp>(initBlock.getTerminator());
auto addrOp = returnOp.getArg().getDefiningOp<AddressOfOp>();
// FIXME: This is a best effort solution. The AliasOp body might be more
// complex and in that case we bail out with success. To completely match
// the LLVM IR logic it would be necessary to implement proper alias and
// cast stripping.
if (!addrOp)
return success();
resolver = addrOp.getFunction(symbolTable);
alias = addrOp.getAlias(symbolTable);
}
if (!resolver)
return emitOpError("must have a function resolver");
Linkage linkage = resolver.getLinkage();
if (resolver.isExternal() || linkage == Linkage::AvailableExternally)
return emitOpError("resolver must be a definition");
if (!isa<LLVMPointerType>(resolver.getFunctionType().getReturnType()))
return emitOpError("resolver must return a pointer");
auto resolverPtr = dyn_cast<LLVMPointerType>(getResolverType());
if (!resolverPtr || resolverPtr.getAddressSpace() != getAddressSpace())
return emitOpError("resolver has incorrect type");
return success();
}
LogicalResult IFuncOp::verify() {
switch (getLinkage()) {
case Linkage::External:
case Linkage::Internal:
case Linkage::Private:
case Linkage::Weak:
case Linkage::WeakODR:
case Linkage::Linkonce:
case Linkage::LinkonceODR:
break;
default:
return emitOpError() << "'" << stringifyLinkage(getLinkage())
<< "' linkage not supported in ifuncs, available "
"options: private, internal, linkonce, weak, "
"linkonce_odr, weak_odr, or external linkage";
}
return success();
}
//===----------------------------------------------------------------------===//
// ShuffleVectorOp
//===----------------------------------------------------------------------===//
void ShuffleVectorOp::build(OpBuilder &builder, OperationState &state, Value v1,
Value v2, DenseI32ArrayAttr mask,
ArrayRef<NamedAttribute> attrs) {
auto containerType = v1.getType();
auto vType = LLVM::getVectorType(
cast<VectorType>(containerType).getElementType(), mask.size(),
LLVM::isScalableVectorType(containerType));
build(builder, state, vType, v1, v2, mask);
state.addAttributes(attrs);
}
void ShuffleVectorOp::build(OpBuilder &builder, OperationState &state, Value v1,
Value v2, ArrayRef<int32_t> mask) {
build(builder, state, v1, v2, builder.getDenseI32ArrayAttr(mask));
}
/// Build the result type of a shuffle vector operation.
static ParseResult parseShuffleType(AsmParser &parser, Type v1Type,
Type &resType, DenseI32ArrayAttr mask) {
if (!LLVM::isCompatibleVectorType(v1Type))
return parser.emitError(parser.getCurrentLocation(),
"expected an LLVM compatible vector type");
resType =
LLVM::getVectorType(cast<VectorType>(v1Type).getElementType(),
mask.size(), LLVM::isScalableVectorType(v1Type));
return success();
}
/// Nothing to do when the result type is inferred.
static void printShuffleType(AsmPrinter &printer, Operation *op, Type v1Type,
Type resType, DenseI32ArrayAttr mask) {}
LogicalResult ShuffleVectorOp::verify() {
if (LLVM::isScalableVectorType(getV1().getType()) &&
llvm::any_of(getMask(), [](int32_t v) { return v != 0; }))
return emitOpError("expected a splat operation for scalable vectors");
return success();
}
// Folding for shufflevector op when v1 is single element 1D vector
// and the mask is a single zero. OpFoldResult will be v1 in this case.
OpFoldResult ShuffleVectorOp::fold(FoldAdaptor adaptor) {
// Check if operand 0 is a single element vector.
auto vecType = llvm::dyn_cast<VectorType>(getV1().getType());
if (!vecType || vecType.getRank() != 1 || vecType.getNumElements() != 1)
return {};
// Check if the mask is a single zero.
// Note: The mask is guaranteed to be non-empty.
if (getMask().size() != 1 || getMask()[0] != 0)
return {};
return getV1();
}
//===----------------------------------------------------------------------===//
// Implementations for LLVM::LLVMFuncOp.
//===----------------------------------------------------------------------===//
// Add the entry block to the function.
Block *LLVMFuncOp::addEntryBlock(OpBuilder &builder) {
assert(empty() && "function already has an entry block");
OpBuilder::InsertionGuard g(builder);
Block *entry = builder.createBlock(&getBody());
// FIXME: Allow passing in proper locations for the entry arguments.
LLVMFunctionType type = getFunctionType();
for (unsigned i = 0, e = type.getNumParams(); i < e; ++i)
entry->addArgument(type.getParamType(i), getLoc());
return entry;
}
void LLVMFuncOp::build(OpBuilder &builder, OperationState &result,
StringRef name, Type type, LLVM::Linkage linkage,
bool dsoLocal, CConv cconv, SymbolRefAttr comdat,
ArrayRef<NamedAttribute> attrs,
ArrayRef<DictionaryAttr> argAttrs,
std::optional<uint64_t> functionEntryCount) {
result.addRegion();
result.addAttribute(SymbolTable::getSymbolAttrName(),
builder.getStringAttr(name));
result.addAttribute(getFunctionTypeAttrName(result.name),
TypeAttr::get(type));
result.addAttribute(getLinkageAttrName(result.name),
LinkageAttr::get(builder.getContext(), linkage));
result.addAttribute(getCConvAttrName(result.name),
CConvAttr::get(builder.getContext(), cconv));
result.attributes.append(attrs.begin(), attrs.end());
if (dsoLocal)
result.addAttribute(getDsoLocalAttrName(result.name),
builder.getUnitAttr());
if (comdat)
result.addAttribute(getComdatAttrName(result.name), comdat);
if (functionEntryCount)
result.addAttribute(getFunctionEntryCountAttrName(result.name),
builder.getI64IntegerAttr(functionEntryCount.value()));
if (argAttrs.empty())
return;
assert(llvm::cast<LLVMFunctionType>(type).getNumParams() == argAttrs.size() &&
"expected as many argument attribute lists as arguments");
call_interface_impl::addArgAndResultAttrs(
builder, result, argAttrs, /*resultAttrs=*/{},
getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
}
// Builds an LLVM function type from the given lists of input and output types.
// Returns a null type if any of the types provided are non-LLVM types, or if
// there is more than one output type.
static Type
buildLLVMFunctionType(OpAsmParser &parser, SMLoc loc, ArrayRef<Type> inputs,
ArrayRef<Type> outputs,
function_interface_impl::VariadicFlag variadicFlag) {
Builder &b = parser.getBuilder();
if (outputs.size() > 1) {
parser.emitError(loc, "failed to construct function type: expected zero or "
"one function result");
return {};
}
// Convert inputs to LLVM types, exit early on error.
SmallVector<Type, 4> llvmInputs;
for (auto t : inputs) {
if (!isCompatibleType(t)) {
parser.emitError(loc, "failed to construct function type: expected LLVM "
"type for function arguments");
return {};
}
llvmInputs.push_back(t);
}
// No output is denoted as "void" in LLVM type system.
Type llvmOutput =
outputs.empty() ? LLVMVoidType::get(b.getContext()) : outputs.front();
if (!isCompatibleType(llvmOutput)) {
parser.emitError(loc, "failed to construct function type: expected LLVM "
"type for function results")
<< llvmOutput;
return {};
}
return LLVMFunctionType::get(llvmOutput, llvmInputs,
variadicFlag.isVariadic());
}
// Parses an LLVM function.
//
// operation ::= `llvm.func` linkage? cconv? function-signature
// (`comdat(` symbol-ref-id `)`)?
// function-attributes?
// function-body
//
ParseResult LLVMFuncOp::parse(OpAsmParser &parser, OperationState &result) {
// Default to external linkage if no keyword is provided.
result.addAttribute(getLinkageAttrName(result.name),
LinkageAttr::get(parser.getContext(),
parseOptionalLLVMKeyword<Linkage>(
parser, LLVM::Linkage::External)));
// Parse optional visibility, default to Default.
result.addAttribute(getVisibility_AttrName(result.name),
parser.getBuilder().getI64IntegerAttr(
parseOptionalLLVMKeyword<LLVM::Visibility, int64_t>(
parser, LLVM::Visibility::Default)));
// Parse optional UnnamedAddr, default to None.
result.addAttribute(getUnnamedAddrAttrName(result.name),
parser.getBuilder().getI64IntegerAttr(
parseOptionalLLVMKeyword<UnnamedAddr, int64_t>(
parser, LLVM::UnnamedAddr::None)));
// Default to C Calling Convention if no keyword is provided.
result.addAttribute(
getCConvAttrName(result.name),
CConvAttr::get(parser.getContext(),
parseOptionalLLVMKeyword<CConv>(parser, LLVM::CConv::C)));
StringAttr nameAttr;
SmallVector<OpAsmParser::Argument> entryArgs;
SmallVector<DictionaryAttr> resultAttrs;
SmallVector<Type> resultTypes;
bool isVariadic;
auto signatureLocation = parser.getCurrentLocation();
if (parser.parseSymbolName(nameAttr, SymbolTable::getSymbolAttrName(),
result.attributes) ||
function_interface_impl::parseFunctionSignatureWithArguments(
parser, /*allowVariadic=*/true, entryArgs, isVariadic, resultTypes,
resultAttrs))
return failure();
SmallVector<Type> argTypes;
for (auto &arg : entryArgs)
argTypes.push_back(arg.type);
auto type =
buildLLVMFunctionType(parser, signatureLocation, argTypes, resultTypes,
function_interface_impl::VariadicFlag(isVariadic));
if (!type)
return failure();
result.addAttribute(getFunctionTypeAttrName(result.name),
TypeAttr::get(type));
if (succeeded(parser.parseOptionalKeyword("vscale_range"))) {
int64_t minRange, maxRange;
if (parser.parseLParen() || parser.parseInteger(minRange) ||
parser.parseComma() || parser.parseInteger(maxRange) ||
parser.parseRParen())
return failure();
auto intTy = IntegerType::get(parser.getContext(), 32);
result.addAttribute(
getVscaleRangeAttrName(result.name),
LLVM::VScaleRangeAttr::get(parser.getContext(),
IntegerAttr::get(intTy, minRange),
IntegerAttr::get(intTy, maxRange)));
}
// Parse the optional comdat selector.
if (succeeded(parser.parseOptionalKeyword("comdat"))) {
SymbolRefAttr comdat;
if (parser.parseLParen() || parser.parseAttribute(comdat) ||
parser.parseRParen())
return failure();
result.addAttribute(getComdatAttrName(result.name), comdat);
}
if (failed(parser.parseOptionalAttrDictWithKeyword(result.attributes)))
return failure();
call_interface_impl::addArgAndResultAttrs(
parser.getBuilder(), result, entryArgs, resultAttrs,
getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
auto *body = result.addRegion();
OptionalParseResult parseResult =
parser.parseOptionalRegion(*body, entryArgs);
return failure(parseResult.has_value() && failed(*parseResult));
}
// Print the LLVMFuncOp. Collects argument and result types and passes them to
// helper functions. Drops "void" result since it cannot be parsed back. Skips
// the external linkage since it is the default value.
void LLVMFuncOp::print(OpAsmPrinter &p) {
p << ' ';
if (getLinkage() != LLVM::Linkage::External)
p << stringifyLinkage(getLinkage()) << ' ';
StringRef visibility = stringifyVisibility(getVisibility_());
if (!visibility.empty())
p << visibility << ' ';
if (auto unnamedAddr = getUnnamedAddr()) {
StringRef str = stringifyUnnamedAddr(*unnamedAddr);
if (!str.empty())
p << str << ' ';
}
if (getCConv() != LLVM::CConv::C)
p << stringifyCConv(getCConv()) << ' ';
p.printSymbolName(getName());
LLVMFunctionType fnType = getFunctionType();
SmallVector<Type, 8> argTypes;
SmallVector<Type, 1> resTypes;
argTypes.reserve(fnType.getNumParams());
for (unsigned i = 0, e = fnType.getNumParams(); i < e; ++i)
argTypes.push_back(fnType.getParamType(i));
Type returnType = fnType.getReturnType();
if (!llvm::isa<LLVMVoidType>(returnType))
resTypes.push_back(returnType);
function_interface_impl::printFunctionSignature(p, *this, argTypes,
isVarArg(), resTypes);
// Print vscale range if present
if (std::optional<VScaleRangeAttr> vscale = getVscaleRange())
p << " vscale_range(" << vscale->getMinRange().getInt() << ", "
<< vscale->getMaxRange().getInt() << ')';
// Print the optional comdat selector.
if (auto comdat = getComdat())
p << " comdat(" << *comdat << ')';
function_interface_impl::printFunctionAttributes(
p, *this,
{getFunctionTypeAttrName(), getArgAttrsAttrName(), getResAttrsAttrName(),
getLinkageAttrName(), getCConvAttrName(), getVisibility_AttrName(),
getComdatAttrName(), getUnnamedAddrAttrName(),
getVscaleRangeAttrName()});
// Print the body if this is not an external function.
Region &body = getBody();
if (!body.empty()) {
p << ' ';
p.printRegion(body, /*printEntryBlockArgs=*/false,
/*printBlockTerminators=*/true);
}
}
// Verifies LLVM- and implementation-specific properties of the LLVM func Op:
// - functions don't have 'common' linkage
// - external functions have 'external' or 'extern_weak' linkage;
// - vararg is (currently) only supported for external functions;
LogicalResult LLVMFuncOp::verify() {
if (getLinkage() == LLVM::Linkage::Common)
return emitOpError() << "functions cannot have '"
<< stringifyLinkage(LLVM::Linkage::Common)
<< "' linkage";
if (failed(verifyComdat(*this, getComdat())))
return failure();
if (isExternal()) {
if (getLinkage() != LLVM::Linkage::External &&
getLinkage() != LLVM::Linkage::ExternWeak)
return emitOpError() << "external functions must have '"
<< stringifyLinkage(LLVM::Linkage::External)
<< "' or '"
<< stringifyLinkage(LLVM::Linkage::ExternWeak)
<< "' linkage";
return success();
}
// In LLVM IR, these attributes are composed by convention, not by design.
if (isNoInline() && isAlwaysInline())
return emitError("no_inline and always_inline attributes are incompatible");
if (isOptimizeNone() && !isNoInline())
return emitOpError("with optimize_none must also be no_inline");
Type landingpadResultTy;
StringRef diagnosticMessage;
bool isLandingpadTypeConsistent =
!walk([&](Operation *op) {
const auto checkType = [&](Type type, StringRef errorMessage) {
if (!landingpadResultTy) {
landingpadResultTy = type;
return WalkResult::advance();
}
if (landingpadResultTy != type) {
diagnosticMessage = errorMessage;
return WalkResult::interrupt();
}
return WalkResult::advance();
};
return TypeSwitch<Operation *, WalkResult>(op)
.Case([&](LandingpadOp landingpad) {
constexpr StringLiteral errorMessage =
"'llvm.landingpad' should have a consistent result type "
"inside a function";
return checkType(landingpad.getType(), errorMessage);
})
.Case([&](ResumeOp resume) {
constexpr StringLiteral errorMessage =
"'llvm.resume' should have a consistent input type inside a "
"function";
return checkType(resume.getValue().getType(), errorMessage);
})
.Default([](auto) { return WalkResult::skip(); });
}).wasInterrupted();
if (!isLandingpadTypeConsistent) {
assert(!diagnosticMessage.empty() &&
"Expecting a non-empty diagnostic message");
return emitError(diagnosticMessage);
}
if (failed(verifyBlockTags(*this)))
return failure();
return success();
}
/// Verifies LLVM- and implementation-specific properties of the LLVM func Op:
/// - entry block arguments are of LLVM types.
LogicalResult LLVMFuncOp::verifyRegions() {
if (isExternal())
return success();
unsigned numArguments = getFunctionType().getNumParams();
Block &entryBlock = front();
for (unsigned i = 0; i < numArguments; ++i) {
Type argType = entryBlock.getArgument(i).getType();
if (!isCompatibleType(argType))
return emitOpError("entry block argument #")
<< i << " is not of LLVM type";
}
return success();
}
Region *LLVMFuncOp::getCallableRegion() {
if (isExternal())
return nullptr;
return &getBody();
}
//===----------------------------------------------------------------------===//
// UndefOp.
//===----------------------------------------------------------------------===//
/// Fold an undef operation to a dedicated undef attribute.
OpFoldResult LLVM::UndefOp::fold(FoldAdaptor) {
return LLVM::UndefAttr::get(getContext());
}
//===----------------------------------------------------------------------===//
// PoisonOp.
//===----------------------------------------------------------------------===//
/// Fold a poison operation to a dedicated poison attribute.
OpFoldResult LLVM::PoisonOp::fold(FoldAdaptor) {
return LLVM::PoisonAttr::get(getContext());
}
//===----------------------------------------------------------------------===//
// ZeroOp.
//===----------------------------------------------------------------------===//
LogicalResult LLVM::ZeroOp::verify() {
if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType()))
if (!targetExtType.hasProperty(LLVM::LLVMTargetExtType::HasZeroInit))
return emitOpError()
<< "target extension type does not support zero-initializer";
return success();
}
/// Fold a zero operation to a builtin zero attribute when possible and fall
/// back to a dedicated zero attribute.
OpFoldResult LLVM::ZeroOp::fold(FoldAdaptor) {
OpFoldResult result = Builder(getContext()).getZeroAttr(getType());
if (result)
return result;
return LLVM::ZeroAttr::get(getContext());
}
//===----------------------------------------------------------------------===//
// ConstantOp.
//===----------------------------------------------------------------------===//
/// Compute the total number of elements in the given type, also taking into
/// account nested types. Supported types are `VectorType` and `LLVMArrayType`.
/// Everything else is treated as a scalar.
static int64_t getNumElements(Type t) {
if (auto vecType = dyn_cast<VectorType>(t)) {
assert(!vecType.isScalable() &&
"number of elements of a scalable vector type is unknown");
return vecType.getNumElements() * getNumElements(vecType.getElementType());
}
if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(t))
return arrayType.getNumElements() *
getNumElements(arrayType.getElementType());
return 1;
}
/// Determine the element type of `type`. Supported types are `VectorType`,
/// `TensorType`, and `LLVMArrayType`. Everything else is treated as a scalar.
static Type getElementType(Type type) {
while (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(type))
type = arrayType.getElementType();
if (auto vecType = dyn_cast<VectorType>(type))
return vecType.getElementType();
if (auto tenType = dyn_cast<TensorType>(type))
return tenType.getElementType();
return type;
}
/// Check if the given type is a scalable vector type or a vector/array type
/// that contains a nested scalable vector type.
static bool hasScalableVectorType(Type t) {
if (auto vecType = dyn_cast<VectorType>(t)) {
if (vecType.isScalable())
return true;
return hasScalableVectorType(vecType.getElementType());
}
if (auto arrayType = dyn_cast<LLVM::LLVMArrayType>(t))
return hasScalableVectorType(arrayType.getElementType());
return false;
}
/// Verifies the constant array represented by `arrayAttr` matches the provided
/// `arrayType`.
static LogicalResult verifyStructArrayConstant(LLVM::ConstantOp op,
LLVM::LLVMArrayType arrayType,
ArrayAttr arrayAttr, int dim) {
if (arrayType.getNumElements() != arrayAttr.size())
return op.emitOpError()
<< "array attribute size does not match array type size in "
"dimension "
<< dim << ": " << arrayAttr.size() << " vs. "
<< arrayType.getNumElements();
llvm::DenseSet<Attribute> elementsVerified;
// Recursively verify sub-dimensions for multidimensional arrays.
if (auto subArrayType =
dyn_cast<LLVM::LLVMArrayType>(arrayType.getElementType())) {
for (auto [idx, elementAttr] : llvm::enumerate(arrayAttr))
if (elementsVerified.insert(elementAttr).second) {
if (isa<LLVM::ZeroAttr, LLVM::UndefAttr>(elementAttr))
continue;
auto subArrayAttr = dyn_cast<ArrayAttr>(elementAttr);
if (!subArrayAttr)
return op.emitOpError()
<< "nested attribute for sub-array in dimension " << dim
<< " at index " << idx
<< " must be a zero, or undef, or array attribute";
if (failed(verifyStructArrayConstant(op, subArrayType, subArrayAttr,
dim + 1)))
return failure();
}
return success();
}
// Forbid usages of ArrayAttr for simple array types that should use
// DenseElementsAttr instead. Note that there would be a use case for such
// array types when one element value is obtained via a ptr-to-int conversion
// from a symbol and cannot be represented in a DenseElementsAttr, but no MLIR
// user needs this so far, and it seems better to avoid people misusing the
// ArrayAttr for simple types.
auto structType = dyn_cast<LLVM::LLVMStructType>(arrayType.getElementType());
if (!structType)
return op.emitOpError() << "for array with an array attribute must have a "
"struct element type";
// Shallow verification that leaf attributes are appropriate as struct initial
// value.
size_t numStructElements = structType.getBody().size();
for (auto [idx, elementAttr] : llvm::enumerate(arrayAttr)) {
if (elementsVerified.insert(elementAttr).second) {
if (isa<LLVM::ZeroAttr, LLVM::UndefAttr>(elementAttr))
continue;
auto subArrayAttr = dyn_cast<ArrayAttr>(elementAttr);
if (!subArrayAttr)
return op.emitOpError()
<< "nested attribute for struct element at index " << idx
<< " must be a zero, or undef, or array attribute";
if (subArrayAttr.size() != numStructElements)
return op.emitOpError()
<< "nested array attribute size for struct element at index "
<< idx << " must match struct size: " << subArrayAttr.size()
<< " vs. " << numStructElements;
}
}
return success();
}
LogicalResult LLVM::ConstantOp::verify() {
if (StringAttr sAttr = llvm::dyn_cast<StringAttr>(getValue())) {
auto arrayType = llvm::dyn_cast<LLVMArrayType>(getType());
if (!arrayType || arrayType.getNumElements() != sAttr.getValue().size() ||
!arrayType.getElementType().isInteger(8)) {
return emitOpError() << "expected array type of "
<< sAttr.getValue().size()
<< " i8 elements for the string constant";
}
return success();
}
if (auto structType = dyn_cast<LLVMStructType>(getType())) {
auto arrayAttr = dyn_cast<ArrayAttr>(getValue());
if (!arrayAttr)
return emitOpError() << "expected array attribute for struct type";
ArrayRef<Type> elementTypes = structType.getBody();
if (arrayAttr.size() != elementTypes.size()) {
return emitOpError() << "expected array attribute of size "
<< elementTypes.size();
}
for (auto [i, attr, type] : llvm::enumerate(arrayAttr, elementTypes)) {
if (!type.isSignlessIntOrIndexOrFloat()) {
return emitOpError() << "expected struct element types to be floating "
"point type or integer type";
}
if (!isa<FloatAttr, IntegerAttr>(attr)) {
return emitOpError() << "expected element of array attribute to be "
"floating point or integer";
}
if (cast<TypedAttr>(attr).getType() != type)
return emitOpError()
<< "struct element at index " << i << " is of wrong type";
}
return success();
}
if (auto targetExtType = dyn_cast<LLVMTargetExtType>(getType()))
return emitOpError() << "does not support target extension type.";
// Check that an attribute whose element type has floating point semantics
// `attributeFloatSemantics` is compatible with a type whose element type
// is `constantElementType`.
//
// Requirement is that either
// 1) They have identical floating point types.
// 2) `constantElementType` is an integer type of the same width as the float
// attribute. This is to support builtin MLIR float types without LLVM
// equivalents, see comments in getLLVMConstant for more details.
auto verifyFloatSemantics =
[this](const llvm::fltSemantics &attributeFloatSemantics,
Type constantElementType) -> LogicalResult {
if (auto floatType = dyn_cast<FloatType>(constantElementType)) {
if (&floatType.getFloatSemantics() != &attributeFloatSemantics) {
return emitOpError()
<< "attribute and type have different float semantics";
}
return success();
}
unsigned floatWidth = APFloat::getSizeInBits(attributeFloatSemantics);
if (isa<IntegerType>(constantElementType)) {
if (!constantElementType.isInteger(floatWidth))
return emitOpError() << "expected integer type of width " << floatWidth;
return success();
}
return success();
};
// Verification of IntegerAttr, FloatAttr, ElementsAttr, ArrayAttr.
if (isa<IntegerAttr>(getValue())) {
if (!llvm::isa<IntegerType>(getType()))
return emitOpError() << "expected integer type";
} else if (auto floatAttr = dyn_cast<FloatAttr>(getValue())) {
return verifyFloatSemantics(floatAttr.getValue().getSemantics(), getType());
} else if (auto elementsAttr = dyn_cast<ElementsAttr>(getValue())) {
if (hasScalableVectorType(getType())) {
// The exact number of elements of a scalable vector is unknown, so we
// allow only splat attributes.
auto splatElementsAttr = dyn_cast<SplatElementsAttr>(getValue());
if (!splatElementsAttr)
return emitOpError()
<< "scalable vector type requires a splat attribute";
return success();
}
if (!isa<VectorType, LLVM::LLVMArrayType>(getType()))
return emitOpError() << "expected vector or array type";
// The number of elements of the attribute and the type must match.
int64_t attrNumElements = elementsAttr.getNumElements();
if (getNumElements(getType()) != attrNumElements) {
return emitOpError()
<< "type and attribute have a different number of elements: "
<< getNumElements(getType()) << " vs. " << attrNumElements;
}
Type attrElmType = getElementType(elementsAttr.getType());
Type resultElmType = getElementType(getType());
if (auto floatType = dyn_cast<FloatType>(attrElmType))
return verifyFloatSemantics(floatType.getFloatSemantics(), resultElmType);
if (isa<IntegerType>(attrElmType) && !isa<IntegerType>(resultElmType)) {
return emitOpError(
"expected integer element type for integer elements attribute");
}
} else if (auto arrayAttr = dyn_cast<ArrayAttr>(getValue())) {
// The case where the constant is LLVMStructType has already been handled.
auto arrayType = dyn_cast<LLVM::LLVMArrayType>(getType());
if (!arrayType)
return emitOpError()
<< "expected array or struct type for array attribute";
// When the attribute is an ArrayAttr, check that its nesting matches the
// corresponding ArrayType or VectorType nesting.
return verifyStructArrayConstant(*this, arrayType, arrayAttr, /*dim=*/0);
} else {
return emitOpError()
<< "only supports integer, float, string or elements attributes";
}
return success();
}
bool LLVM::ConstantOp::isBuildableWith(Attribute value, Type type) {
// The value's type must be the same as the provided type.
auto typedAttr = dyn_cast<TypedAttr>(value);
if (!typedAttr || typedAttr.getType() != type || !isCompatibleType(type))
return false;
// The value's type must be an LLVM compatible type.
if (!isCompatibleType(type))
return false;
// TODO: Add support for additional attributes kinds once needed.
return isa<IntegerAttr, FloatAttr, ElementsAttr>(value);
}
ConstantOp LLVM::ConstantOp::materialize(OpBuilder &builder, Attribute value,
Type type, Location loc) {
if (isBuildableWith(value, type))
return LLVM::ConstantOp::create(builder, loc, cast<TypedAttr>(value));
return nullptr;
}
// Constant op constant-folds to its value.
OpFoldResult LLVM::ConstantOp::fold(FoldAdaptor) { return getValue(); }
//===----------------------------------------------------------------------===//
// AtomicRMWOp
//===----------------------------------------------------------------------===//
void AtomicRMWOp::build(OpBuilder &builder, OperationState &state,
AtomicBinOp binOp, Value ptr, Value val,
AtomicOrdering ordering, StringRef syncscope,
unsigned alignment, bool isVolatile) {
build(builder, state, val.getType(), binOp, ptr, val, ordering,
!syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isVolatile,
/*access_groups=*/nullptr,
/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}
LogicalResult AtomicRMWOp::verify() {
auto valType = getVal().getType();
if (getBinOp() == AtomicBinOp::fadd || getBinOp() == AtomicBinOp::fsub ||
getBinOp() == AtomicBinOp::fmin || getBinOp() == AtomicBinOp::fmax ||
getBinOp() == AtomicBinOp::fminimum ||
getBinOp() == AtomicBinOp::fmaximum) {
if (isCompatibleVectorType(valType)) {
if (isScalableVectorType(valType))
return emitOpError("expected LLVM IR fixed vector type");
Type elemType = llvm::cast<VectorType>(valType).getElementType();
if (!isCompatibleFloatingPointType(elemType))
return emitOpError(
"expected LLVM IR floating point type for vector element");
} else if (!isCompatibleFloatingPointType(valType)) {
return emitOpError("expected LLVM IR floating point type");
}
} else if (getBinOp() == AtomicBinOp::xchg) {
DataLayout dataLayout = DataLayout::closest(*this);
if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
return emitOpError("unexpected LLVM IR type for 'xchg' bin_op");
} else {
auto intType = llvm::dyn_cast<IntegerType>(valType);
unsigned intBitWidth = intType ? intType.getWidth() : 0;
if (intBitWidth != 8 && intBitWidth != 16 && intBitWidth != 32 &&
intBitWidth != 64)
return emitOpError("expected LLVM IR integer type");
}
if (static_cast<unsigned>(getOrdering()) <
static_cast<unsigned>(AtomicOrdering::monotonic))
return emitOpError() << "expected at least '"
<< stringifyAtomicOrdering(AtomicOrdering::monotonic)
<< "' ordering";
return success();
}
//===----------------------------------------------------------------------===//
// AtomicCmpXchgOp
//===----------------------------------------------------------------------===//
/// Returns an LLVM struct type that contains a value type and a boolean type.
static LLVMStructType getValAndBoolStructType(Type valType) {
auto boolType = IntegerType::get(valType.getContext(), 1);
return LLVMStructType::getLiteral(valType.getContext(), {valType, boolType});
}
void AtomicCmpXchgOp::build(OpBuilder &builder, OperationState &state,
Value ptr, Value cmp, Value val,
AtomicOrdering successOrdering,
AtomicOrdering failureOrdering, StringRef syncscope,
unsigned alignment, bool isWeak, bool isVolatile) {
build(builder, state, getValAndBoolStructType(val.getType()), ptr, cmp, val,
successOrdering, failureOrdering,
!syncscope.empty() ? builder.getStringAttr(syncscope) : nullptr,
alignment ? builder.getI64IntegerAttr(alignment) : nullptr, isWeak,
isVolatile, /*access_groups=*/nullptr,
/*alias_scopes=*/nullptr, /*noalias_scopes=*/nullptr, /*tbaa=*/nullptr);
}
LogicalResult AtomicCmpXchgOp::verify() {
auto ptrType = llvm::cast<LLVM::LLVMPointerType>(getPtr().getType());
if (!ptrType)
return emitOpError("expected LLVM IR pointer type for operand #0");
auto valType = getVal().getType();
DataLayout dataLayout = DataLayout::closest(*this);
if (!isTypeCompatibleWithAtomicOp(valType, dataLayout))
return emitOpError("unexpected LLVM IR type");
if (getSuccessOrdering() < AtomicOrdering::monotonic ||
getFailureOrdering() < AtomicOrdering::monotonic)
return emitOpError("ordering must be at least 'monotonic'");
if (getFailureOrdering() == AtomicOrdering::release ||
getFailureOrdering() == AtomicOrdering::acq_rel)
return emitOpError("failure ordering cannot be 'release' or 'acq_rel'");
return success();
}
//===----------------------------------------------------------------------===//
// FenceOp
//===----------------------------------------------------------------------===//
void FenceOp::build(OpBuilder &builder, OperationState &state,
AtomicOrdering ordering, StringRef syncscope) {
build(builder, state, ordering,
syncscope.empty() ? nullptr : builder.getStringAttr(syncscope));
}
LogicalResult FenceOp::verify() {
if (getOrdering() == AtomicOrdering::not_atomic ||
getOrdering() == AtomicOrdering::unordered ||
getOrdering() == AtomicOrdering::monotonic)
return emitOpError("can be given only acquire, release, acq_rel, "
"and seq_cst orderings");
return success();
}
//===----------------------------------------------------------------------===//
// Verifier for extension ops
//===----------------------------------------------------------------------===//
/// Verifies that the given extension operation operates on consistent scalars
/// or vectors, and that the target width is larger than the input width.
template <class ExtOp>
static LogicalResult verifyExtOp(ExtOp op) {
IntegerType inputType, outputType;
if (isCompatibleVectorType(op.getArg().getType())) {
if (!isCompatibleVectorType(op.getResult().getType()))
return op.emitError(
"input type is a vector but output type is an integer");
if (getVectorNumElements(op.getArg().getType()) !=
getVectorNumElements(op.getResult().getType()))
return op.emitError("input and output vectors are of incompatible shape");
// Because this is a CastOp, the element of vectors is guaranteed to be an
// integer.
inputType = cast<IntegerType>(
cast<VectorType>(op.getArg().getType()).getElementType());
outputType = cast<IntegerType>(
cast<VectorType>(op.getResult().getType()).getElementType());
} else {
// Because this is a CastOp and arg is not a vector, arg is guaranteed to be
// an integer.
inputType = cast<IntegerType>(op.getArg().getType());
outputType = dyn_cast<IntegerType>(op.getResult().getType());
if (!outputType)
return op.emitError(
"input type is an integer but output type is a vector");
}
if (outputType.getWidth() <= inputType.getWidth())
return op.emitError("integer width of the output type is smaller or "
"equal to the integer width of the input type");
return success();
}
//===----------------------------------------------------------------------===//
// ZExtOp
//===----------------------------------------------------------------------===//
LogicalResult ZExtOp::verify() { return verifyExtOp<ZExtOp>(*this); }
OpFoldResult LLVM::ZExtOp::fold(FoldAdaptor adaptor) {
auto arg = dyn_cast_or_null<IntegerAttr>(adaptor.getArg());
if (!arg)
return {};
size_t targetSize = cast<IntegerType>(getType()).getWidth();
return IntegerAttr::get(getType(), arg.getValue().zext(targetSize));
}
//===----------------------------------------------------------------------===//
// SExtOp
//===----------------------------------------------------------------------===//
LogicalResult SExtOp::verify() { return verifyExtOp<SExtOp>(*this); }
//===----------------------------------------------------------------------===//
// Folder and verifier for LLVM::BitcastOp
//===----------------------------------------------------------------------===//
/// Folds a cast op that can be chained.
template <typename T>
static OpFoldResult foldChainableCast(T castOp,
typename T::FoldAdaptor adaptor) {
// cast(x : T0, T0) -> x
if (castOp.getArg().getType() == castOp.getType())
return castOp.getArg();
if (auto prev = castOp.getArg().template getDefiningOp<T>()) {
// cast(cast(x : T0, T1), T0) -> x
if (prev.getArg().getType() == castOp.getType())
return prev.getArg();
// cast(cast(x : T0, T1), T2) -> cast(x: T0, T2)
castOp.getArgMutable().set(prev.getArg());
return Value{castOp};
}
return {};
}
OpFoldResult LLVM::BitcastOp::fold(FoldAdaptor adaptor) {
return foldChainableCast(*this, adaptor);
}
LogicalResult LLVM::BitcastOp::verify() {
auto resultType = llvm::dyn_cast<LLVMPointerType>(
extractVectorElementType(getResult().getType()));
auto sourceType = llvm::dyn_cast<LLVMPointerType>(
extractVectorElementType(getArg().getType()));
// If one of the types is a pointer (or vector of pointers), then
// both source and result type have to be pointers.
if (static_cast<bool>(resultType) != static_cast<bool>(sourceType))
return emitOpError("can only cast pointers from and to pointers");
if (!resultType)
return success();
auto isVector = llvm::IsaPred<VectorType>;
// Due to bitcast requiring both operands to be of the same size, it is not
// possible for only one of the two to be a pointer of vectors.
if (isVector(getResult().getType()) && !isVector(getArg().getType()))
return emitOpError("cannot cast pointer to vector of pointers");
if (!isVector(getResult().getType()) && isVector(getArg().getType()))
return emitOpError("cannot cast vector of pointers to pointer");
// Bitcast cannot cast between pointers of different address spaces.
// 'llvm.addrspacecast' must be used for this purpose instead.
if (resultType.getAddressSpace() != sourceType.getAddressSpace())
return emitOpError("cannot cast pointers of different address spaces, "
"use 'llvm.addrspacecast' instead");
return success();
}
//===----------------------------------------------------------------------===//
// Folder for LLVM::AddrSpaceCastOp
//===----------------------------------------------------------------------===//
OpFoldResult LLVM::AddrSpaceCastOp::fold(FoldAdaptor adaptor) {
return foldChainableCast(*this, adaptor);
}
Value LLVM::AddrSpaceCastOp::getViewSource() { return getArg(); }
//===----------------------------------------------------------------------===//
// Folder for LLVM::GEPOp
//===----------------------------------------------------------------------===//
OpFoldResult LLVM::GEPOp::fold(FoldAdaptor adaptor) {
GEPIndicesAdaptor<ArrayRef<Attribute>> indices(getRawConstantIndicesAttr(),
adaptor.getDynamicIndices());
// gep %x:T, 0 -> %x
if (getBase().getType() == getType() && indices.size() == 1)
if (auto integer = llvm::dyn_cast_or_null<IntegerAttr>(indices[0]))
if (integer.getValue().isZero())
return getBase();
// Canonicalize any dynamic indices of constant value to constant indices.
bool changed = false;
SmallVector<GEPArg> gepArgs;
for (auto iter : llvm::enumerate(indices)) {
auto integer = llvm::dyn_cast_or_null<IntegerAttr>(iter.value());
// Constant indices can only be int32_t, so if integer does not fit we
// are forced to keep it dynamic, despite being a constant.
if (!indices.isDynamicIndex(iter.index()) || !integer ||
!integer.getValue().isSignedIntN(kGEPConstantBitWidth)) {
PointerUnion<IntegerAttr, Value> existing = getIndices()[iter.index()];
if (Value val = llvm::dyn_cast_if_present<Value>(existing))
gepArgs.emplace_back(val);
else
gepArgs.emplace_back(cast<IntegerAttr>(existing).getInt());
continue;
}
changed = true;
gepArgs.emplace_back(integer.getInt());
}
if (changed) {
SmallVector<int32_t> rawConstantIndices;
SmallVector<Value> dynamicIndices;
destructureIndices(getElemType(), gepArgs, rawConstantIndices,
dynamicIndices);
getDynamicIndicesMutable().assign(dynamicIndices);
setRawConstantIndices(rawConstantIndices);
return Value{*this};
}
return {};
}
Value LLVM::GEPOp::getViewSource() { return getBase(); }
//===----------------------------------------------------------------------===//
// ShlOp
//===----------------------------------------------------------------------===//
OpFoldResult LLVM::ShlOp::fold(FoldAdaptor adaptor) {
auto rhs = dyn_cast_or_null<IntegerAttr>(adaptor.getRhs());
if (!rhs)
return {};
if (rhs.getValue().getZExtValue() >=
getLhs().getType().getIntOrFloatBitWidth())
return {}; // TODO: Fold into poison.
auto lhs = dyn_cast_or_null<IntegerAttr>(adaptor.getLhs());
if (!lhs)
return {};
return IntegerAttr::get(getType(), lhs.getValue().shl(rhs.getValue()));
}
//===----------------------------------------------------------------------===//
// OrOp
//===----------------------------------------------------------------------===//
OpFoldResult LLVM::OrOp::fold(FoldAdaptor adaptor) {
auto lhs = dyn_cast_or_null<IntegerAttr>(adaptor.getLhs());
if (!lhs)
return {};
auto rhs = dyn_cast_or_null<IntegerAttr>(adaptor.getRhs());
if (!rhs)
return {};
return IntegerAttr::get(getType(), lhs.getValue() | rhs.getValue());
}
//===----------------------------------------------------------------------===//
// CallIntrinsicOp
//===----------------------------------------------------------------------===//
LogicalResult CallIntrinsicOp::verify() {
if (!getIntrin().starts_with("llvm."))
return emitOpError() << "intrinsic name must start with 'llvm.'";
if (failed(verifyOperandBundles(*this)))
return failure();
return success();
}
void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
mlir::StringAttr intrin, mlir::ValueRange args) {
build(builder, state, /*resultTypes=*/TypeRange{}, intrin, args,
FastmathFlagsAttr{},
/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
/*res_attrs=*/{});
}
void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
mlir::StringAttr intrin, mlir::ValueRange args,
mlir::LLVM::FastmathFlagsAttr fastMathFlags) {
build(builder, state, /*resultTypes=*/TypeRange{}, intrin, args,
fastMathFlags,
/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
/*res_attrs=*/{});
}
void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
mlir::Type resultType, mlir::StringAttr intrin,
mlir::ValueRange args) {
build(builder, state, {resultType}, intrin, args, FastmathFlagsAttr{},
/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
/*res_attrs=*/{});
}
void CallIntrinsicOp::build(OpBuilder &builder, OperationState &state,
mlir::TypeRange resultTypes,
mlir::StringAttr intrin, mlir::ValueRange args,
mlir::LLVM::FastmathFlagsAttr fastMathFlags) {
build(builder, state, resultTypes, intrin, args, fastMathFlags,
/*op_bundle_operands=*/{}, /*op_bundle_tags=*/{}, /*arg_attrs=*/{},
/*res_attrs=*/{});
}
ParseResult CallIntrinsicOp::parse(OpAsmParser &parser,
OperationState &result) {
StringAttr intrinAttr;
SmallVector<OpAsmParser::UnresolvedOperand, 4> operands;
SmallVector<SmallVector<OpAsmParser::UnresolvedOperand>> opBundleOperands;
SmallVector<SmallVector<Type>> opBundleOperandTypes;
ArrayAttr opBundleTags;
// Parse intrinsic name.
if (parser.parseCustomAttributeWithFallback(
intrinAttr, parser.getBuilder().getType<NoneType>()))
return failure();
result.addAttribute(CallIntrinsicOp::getIntrinAttrName(result.name),
intrinAttr);
if (parser.parseLParen())
return failure();
// Parse the function arguments.
if (parser.parseOperandList(operands))
return mlir::failure();
if (parser.parseRParen())
return mlir::failure();
// Handle bundles.
SMLoc opBundlesLoc = parser.getCurrentLocation();
if (std::optional<ParseResult> result = parseOpBundles(
parser, opBundleOperands, opBundleOperandTypes, opBundleTags);
result && failed(*result))
return failure();
if (opBundleTags && !opBundleTags.empty())
result.addAttribute(
CallIntrinsicOp::getOpBundleTagsAttrName(result.name).getValue(),
opBundleTags);
if (parser.parseOptionalAttrDict(result.attributes))
return mlir::failure();
SmallVector<DictionaryAttr> argAttrs;
SmallVector<DictionaryAttr> resultAttrs;
if (parseCallTypeAndResolveOperands(parser, result, /*isDirect=*/true,
operands, argAttrs, resultAttrs))
return failure();
call_interface_impl::addArgAndResultAttrs(
parser.getBuilder(), result, argAttrs, resultAttrs,
getArgAttrsAttrName(result.name), getResAttrsAttrName(result.name));
if (resolveOpBundleOperands(parser, opBundlesLoc, result, opBundleOperands,
opBundleOperandTypes,
getOpBundleSizesAttrName(result.name)))
return failure();
int32_t numOpBundleOperands = 0;
for (const auto &operands : opBundleOperands)
numOpBundleOperands += operands.size();
result.addAttribute(
CallIntrinsicOp::getOperandSegmentSizeAttr(),
parser.getBuilder().getDenseI32ArrayAttr(
{static_cast<int32_t>(operands.size()), numOpBundleOperands}));
return mlir::success();
}
void CallIntrinsicOp::print(OpAsmPrinter &p) {
p << ' ';
p.printAttributeWithoutType(getIntrinAttr());
OperandRange args = getArgs();
p << "(" << args << ")";
// Operand bundles.
if (!getOpBundleOperands().empty()) {
p << ' ';
printOpBundles(p, *this, getOpBundleOperands(),
getOpBundleOperands().getTypes(), getOpBundleTagsAttr());
}
p.printOptionalAttrDict(processFMFAttr((*this)->getAttrs()),
{getOperandSegmentSizesAttrName(),
getOpBundleSizesAttrName(), getIntrinAttrName(),
getOpBundleTagsAttrName(), getArgAttrsAttrName(),
getResAttrsAttrName()});
p << " : ";
// Reconstruct the MLIR function type from operand and result types.
call_interface_impl::printFunctionSignature(
p, args.getTypes(), getArgAttrsAttr(),
/*isVariadic=*/false, getResultTypes(), getResAttrsAttr());
}
//===----------------------------------------------------------------------===//
// LinkerOptionsOp
//===----------------------------------------------------------------------===//
LogicalResult LinkerOptionsOp::verify() {
if (mlir::Operation *parentOp = (*this)->getParentOp();
parentOp && !satisfiesLLVMModule(parentOp))
return emitOpError("must appear at the module level");
return success();
}
//===----------------------------------------------------------------------===//
// ModuleFlagsOp
//===----------------------------------------------------------------------===//
LogicalResult ModuleFlagsOp::verify() {
if (Operation *parentOp = (*this)->getParentOp();
parentOp && !satisfiesLLVMModule(parentOp))
return emitOpError("must appear at the module level");
for (Attribute flag : getFlags())
if (!isa<ModuleFlagAttr>(flag))
return emitOpError("expected a module flag attribute");
return success();
}
//===----------------------------------------------------------------------===//
// InlineAsmOp
//===----------------------------------------------------------------------===//
void InlineAsmOp::getEffects(
SmallVectorImpl<SideEffects::EffectInstance<MemoryEffects::Effect>>
&effects) {
if (getHasSideEffects()) {
effects.emplace_back(MemoryEffects::Write::get());
effects.emplace_back(MemoryEffects::Read::get());
}
}
//===----------------------------------------------------------------------===//
// BlockAddressOp
//===----------------------------------------------------------------------===//
LogicalResult
BlockAddressOp::verifySymbolUses(SymbolTableCollection &symbolTable) {
Operation *symbol = symbolTable.lookupSymbolIn(parentLLVMModule(*this),
getBlockAddr().getFunction());
auto function = dyn_cast_or_null<LLVMFuncOp>(symbol);
if (!function)
return emitOpError("must reference a function defined by 'llvm.func'");
return success();
}
LLVMFuncOp BlockAddressOp::getFunction(SymbolTableCollection &symbolTable) {
return dyn_cast_or_null<LLVMFuncOp>(symbolTable.lookupSymbolIn(
parentLLVMModule(*this), getBlockAddr().getFunction()));
}
BlockTagOp BlockAddressOp::getBlockTagOp() {
auto funcOp = dyn_cast<LLVMFuncOp>(mlir::SymbolTable::lookupNearestSymbolFrom(
parentLLVMModule(*this), getBlockAddr().getFunction()));
if (!funcOp)
return nullptr;
BlockTagOp blockTagOp = nullptr;
funcOp.walk([&](LLVM::BlockTagOp labelOp) {
if (labelOp.getTag() == getBlockAddr().getTag()) {
blockTagOp = labelOp;
return WalkResult::interrupt();
}
return WalkResult::advance();
});
return blockTagOp;
}
LogicalResult BlockAddressOp::verify() {
if (!getBlockTagOp())
return emitOpError(
"expects an existing block label target in the referenced function");
return success();
}
/// Fold a blockaddress operation to a dedicated blockaddress
/// attribute.
OpFoldResult BlockAddressOp::fold(FoldAdaptor) { return getBlockAddr(); }
//===----------------------------------------------------------------------===//
// LLVM::IndirectBrOp
//===----------------------------------------------------------------------===//
SuccessorOperands IndirectBrOp::getSuccessorOperands(unsigned index) {
assert(index < getNumSuccessors() && "invalid successor index");
return SuccessorOperands(getSuccOperandsMutable()[index]);
}
void IndirectBrOp::build(OpBuilder &odsBuilder, OperationState &odsState,
Value addr, ArrayRef<ValueRange> succOperands,
BlockRange successors) {
odsState.addOperands(addr);
for (ValueRange range : succOperands)
odsState.addOperands(range);
SmallVector<int32_t> rangeSegments;
for (ValueRange range : succOperands)
rangeSegments.push_back(range.size());
odsState.getOrAddProperties<Properties>().indbr_operand_segments =
odsBuilder.getDenseI32ArrayAttr(rangeSegments);
odsState.addSuccessors(successors);
}
static ParseResult parseIndirectBrOpSucessors(
OpAsmParser &parser, Type &flagType,
SmallVectorImpl<Block *> &succOperandBlocks,
SmallVectorImpl<SmallVector<OpAsmParser::UnresolvedOperand>> &succOperands,
SmallVectorImpl<SmallVector<Type>> &succOperandsTypes) {
if (failed(parser.parseCommaSeparatedList(
OpAsmParser::Delimiter::Square,
[&]() {
Block *destination = nullptr;
SmallVector<OpAsmParser::UnresolvedOperand> operands;
SmallVector<Type> operandTypes;
if (parser.parseSuccessor(destination).failed())
return failure();
if (succeeded(parser.parseOptionalLParen())) {
if (failed(parser.parseOperandList(
operands, OpAsmParser::Delimiter::None)) ||
failed(parser.parseColonTypeList(operandTypes)) ||
failed(parser.parseRParen()))
return failure();
}
succOperandBlocks.push_back(destination);
succOperands.emplace_back(operands);
succOperandsTypes.emplace_back(operandTypes);
return success();
},
"successor blocks")))
return failure();
return success();
}
static void
printIndirectBrOpSucessors(OpAsmPrinter &p, IndirectBrOp op, Type flagType,
SuccessorRange succs, OperandRangeRange succOperands,
const TypeRangeRange &succOperandsTypes) {
p << "[";
llvm::interleave(
llvm::zip(succs, succOperands),
[&](auto i) {
p.printNewline();
p.printSuccessorAndUseList(std::get<0>(i), std::get<1>(i));
},
[&] { p << ','; });
if (!succOperands.empty())
p.printNewline();
p << "]";
}
//===----------------------------------------------------------------------===//
// SincosOp (intrinsic)
//===----------------------------------------------------------------------===//
LogicalResult LLVM::SincosOp::verify() {
auto operandType = getOperand().getType();
auto resultType = getResult().getType();
auto resultStructType =
mlir::dyn_cast<mlir::LLVM::LLVMStructType>(resultType);
if (!resultStructType || resultStructType.getBody().size() != 2 ||
resultStructType.getBody()[0] != operandType ||
resultStructType.getBody()[1] != operandType) {
return emitOpError("expected result type to be an homogeneous struct with "
"two elements matching the operand type, but got ")
<< resultType;
}
return success();
}
//===----------------------------------------------------------------------===//
// AssumeOp (intrinsic)
//===----------------------------------------------------------------------===//
void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
mlir::Value cond) {
return build(builder, state, cond, /*op_bundle_operands=*/{},
/*op_bundle_tags=*/ArrayAttr{});
}
void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
Value cond, llvm::StringRef tag, ValueRange args) {
return build(builder, state, cond, ArrayRef<ValueRange>(args),
builder.getStrArrayAttr(tag));
}
void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
Value cond, AssumeAlignTag, Value ptr, Value align) {
return build(builder, state, cond, "align", ValueRange{ptr, align});
}
void LLVM::AssumeOp::build(OpBuilder &builder, OperationState &state,
Value cond, AssumeSeparateStorageTag, Value ptr1,
Value ptr2) {
return build(builder, state, cond, "separate_storage",
ValueRange{ptr1, ptr2});
}
LogicalResult LLVM::AssumeOp::verify() { return verifyOperandBundles(*this); }
//===----------------------------------------------------------------------===//
// masked_gather (intrinsic)
//===----------------------------------------------------------------------===//
LogicalResult LLVM::masked_gather::verify() {
auto ptrsVectorType = getPtrs().getType();
Type expectedPtrsVectorType =
LLVM::getVectorType(extractVectorElementType(ptrsVectorType),
LLVM::getVectorNumElements(getRes().getType()));
// Vector of pointers type should match result vector type, other than the
// element type.
if (ptrsVectorType != expectedPtrsVectorType)
return emitOpError("expected operand #1 type to be ")
<< expectedPtrsVectorType;
return success();
}
//===----------------------------------------------------------------------===//
// masked_scatter (intrinsic)
//===----------------------------------------------------------------------===//
LogicalResult LLVM::masked_scatter::verify() {
auto ptrsVectorType = getPtrs().getType();
Type expectedPtrsVectorType =
LLVM::getVectorType(extractVectorElementType(ptrsVectorType),
LLVM::getVectorNumElements(getValue().getType()));
// Vector of pointers type should match value vector type, other than the
// element type.
if (ptrsVectorType != expectedPtrsVectorType)
return emitOpError("expected operand #2 type to be ")
<< expectedPtrsVectorType;
return success();
}
//===----------------------------------------------------------------------===//
// masked_expandload (intrinsic)
//===----------------------------------------------------------------------===//
void LLVM::masked_expandload::build(OpBuilder &builder, OperationState &state,
mlir::TypeRange resTys, Value ptr,
Value mask, Value passthru,
uint64_t align) {
ArrayAttr argAttrs = getLLVMAlignParamForCompressExpand(builder, true, align);
build(builder, state, resTys, ptr, mask, passthru, /*arg_attrs=*/argAttrs,
/*res_attrs=*/nullptr);
}
//===----------------------------------------------------------------------===//
// masked_compressstore (intrinsic)
//===----------------------------------------------------------------------===//
void LLVM::masked_compressstore::build(OpBuilder &builder,
OperationState &state, Value value,
Value ptr, Value mask, uint64_t align) {
ArrayAttr argAttrs =
getLLVMAlignParamForCompressExpand(builder, false, align);
build(builder, state, value, ptr, mask, /*arg_attrs=*/argAttrs,
/*res_attrs=*/nullptr);
}
//===----------------------------------------------------------------------===//
// InlineAsmOp
//===----------------------------------------------------------------------===//
LogicalResult InlineAsmOp::verify() {
if (!getTailCallKindAttr())
return success();
if (getTailCallKindAttr().getTailCallKind() == TailCallKind::MustTail)
return emitOpError(
"tail call kind 'musttail' is not supported by this operation");
return success();
}
//===----------------------------------------------------------------------===//
// UDivOp
//===----------------------------------------------------------------------===//
Speculation::Speculatability UDivOp::getSpeculatability() {
// X / 0 => UB
Value divisor = getRhs();
if (matchPattern(divisor, m_IntRangeWithoutZeroU()))
return Speculation::Speculatable;
return Speculation::NotSpeculatable;
}
//===----------------------------------------------------------------------===//
// SDivOp
//===----------------------------------------------------------------------===//
Speculation::Speculatability SDivOp::getSpeculatability() {
// This function conservatively assumes that all signed division by -1 are
// not speculatable.
// X / 0 => UB
// INT_MIN / -1 => UB
Value divisor = getRhs();
if (matchPattern(divisor, m_IntRangeWithoutZeroS()) &&
matchPattern(divisor, m_IntRangeWithoutNegOneS()))
return Speculation::Speculatable;
return Speculation::NotSpeculatable;
}
//===----------------------------------------------------------------------===//
// LLVMDialect initialization, type parsing, and registration.
//===----------------------------------------------------------------------===//
void LLVMDialect::initialize() {
registerAttributes();
// clang-format off
addTypes<LLVMVoidType,
LLVMTokenType,
LLVMLabelType,
LLVMMetadataType>();
// clang-format on
registerTypes();
addOperations<
#define GET_OP_LIST
#include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
,
#define GET_OP_LIST
#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
>();
// Support unknown operations because not all LLVM operations are registered.
allowUnknownOperations();
declarePromisedInterface<DialectInlinerInterface, LLVMDialect>();
detail::addBytecodeInterface(this);
}
#define GET_OP_CLASSES
#include "mlir/Dialect/LLVMIR/LLVMOps.cpp.inc"
#define GET_OP_CLASSES
#include "mlir/Dialect/LLVMIR/LLVMIntrinsicOps.cpp.inc"
LogicalResult LLVMDialect::verifyDataLayoutString(
StringRef descr, llvm::function_ref<void(const Twine &)> reportError) {
llvm::Expected<llvm::DataLayout> maybeDataLayout =
llvm::DataLayout::parse(descr);
if (maybeDataLayout)
return success();
std::string message;
llvm::raw_string_ostream messageStream(message);
llvm::logAllUnhandledErrors(maybeDataLayout.takeError(), messageStream);
reportError("invalid data layout descriptor: " + message);
return failure();
}
/// Verify LLVM dialect attributes.
LogicalResult LLVMDialect::verifyOperationAttribute(Operation *op,
NamedAttribute attr) {
// If the data layout attribute is present, it must use the LLVM data layout
// syntax. Try parsing it and report errors in case of failure. Users of this
// attribute may assume it is well-formed and can pass it to the (asserting)
// llvm::DataLayout constructor.
if (attr.getName() != LLVM::LLVMDialect::getDataLayoutAttrName())
return success();
if (auto stringAttr = llvm::dyn_cast<StringAttr>(attr.getValue()))
return verifyDataLayoutString(
stringAttr.getValue(),
[op](const Twine &message) { op->emitOpError() << message.str(); });
return op->emitOpError() << "expected '"
<< LLVM::LLVMDialect::getDataLayoutAttrName()
<< "' to be a string attributes";
}
LogicalResult LLVMDialect::verifyParameterAttribute(Operation *op,
Type paramType,
NamedAttribute paramAttr) {
// LLVM attribute may be attached to a result of operation that has not been
// converted to LLVM dialect yet, so the result may have a type with unknown
// representation in LLVM dialect type space. In this case we cannot verify
// whether the attribute may be
bool verifyValueType = isCompatibleType(paramType);
StringAttr name = paramAttr.getName();
auto checkUnitAttrType = [&]() -> LogicalResult {
if (!llvm::isa<UnitAttr>(paramAttr.getValue()))
return op->emitError() << name << " should be a unit attribute";
return success();
};
auto checkTypeAttrType = [&]() -> LogicalResult {
if (!llvm::isa<TypeAttr>(paramAttr.getValue()))
return op->emitError() << name << " should be a type attribute";
return success();
};
auto checkIntegerAttrType = [&]() -> LogicalResult {
if (!llvm::isa<IntegerAttr>(paramAttr.getValue()))
return op->emitError() << name << " should be an integer attribute";
return success();
};
auto checkPointerType = [&]() -> LogicalResult {
if (!llvm::isa<LLVMPointerType>(paramType))
return op->emitError()
<< name << " attribute attached to non-pointer LLVM type";
return success();
};
auto checkIntegerType = [&]() -> LogicalResult {
if (!llvm::isa<IntegerType>(paramType))
return op->emitError()
<< name << " attribute attached to non-integer LLVM type";
return success();
};
auto checkPointerTypeMatches = [&]() -> LogicalResult {
if (failed(checkPointerType()))
return failure();
return success();
};
// Check a unit attribute that is attached to a pointer value.
if (name == LLVMDialect::getNoAliasAttrName() ||
name == LLVMDialect::getReadonlyAttrName() ||
name == LLVMDialect::getReadnoneAttrName() ||
name == LLVMDialect::getWriteOnlyAttrName() ||
name == LLVMDialect::getNestAttrName() ||
name == LLVMDialect::getNoCaptureAttrName() ||
name == LLVMDialect::getNoFreeAttrName() ||
name == LLVMDialect::getNonNullAttrName()) {
if (failed(checkUnitAttrType()))
return failure();
if (verifyValueType && failed(checkPointerType()))
return failure();
return success();
}
// Check a type attribute that is attached to a pointer value.
if (name == LLVMDialect::getStructRetAttrName() ||
name == LLVMDialect::getByValAttrName() ||
name == LLVMDialect::getByRefAttrName() ||
name == LLVMDialect::getElementTypeAttrName() ||
name == LLVMDialect::getInAllocaAttrName() ||
name == LLVMDialect::getPreallocatedAttrName()) {
if (failed(checkTypeAttrType()))
return failure();
if (verifyValueType && failed(checkPointerTypeMatches()))
return failure();
return success();
}
// Check a unit attribute that is attached to an integer value.
if (name == LLVMDialect::getSExtAttrName() ||
name == LLVMDialect::getZExtAttrName()) {
if (failed(checkUnitAttrType()))
return failure();
if (verifyValueType && failed(checkIntegerType()))
return failure();
return success();
}
// Check an integer attribute that is attached to a pointer value.
if (name == LLVMDialect::getAlignAttrName() ||
name == LLVMDialect::getDereferenceableAttrName() ||
name == LLVMDialect::getDereferenceableOrNullAttrName()) {
if (failed(checkIntegerAttrType()))
return failure();
if (verifyValueType && failed(checkPointerType()))
return failure();
return success();
}
// Check an integer attribute that is attached to a pointer value.
if (name == LLVMDialect::getStackAlignmentAttrName()) {
if (failed(checkIntegerAttrType()))
return failure();
return success();
}
// Check a unit attribute that can be attached to arbitrary types.
if (name == LLVMDialect::getNoUndefAttrName() ||
name == LLVMDialect::getInRegAttrName() ||
name == LLVMDialect::getReturnedAttrName())
return checkUnitAttrType();
return success();
}
/// Verify LLVMIR function argument attributes.
LogicalResult LLVMDialect::verifyRegionArgAttribute(Operation *op,
unsigned regionIdx,
unsigned argIdx,
NamedAttribute argAttr) {
auto funcOp = dyn_cast<FunctionOpInterface>(op);
if (!funcOp)
return success();
Type argType = funcOp.getArgumentTypes()[argIdx];
return verifyParameterAttribute(op, argType, argAttr);
}
LogicalResult LLVMDialect::verifyRegionResultAttribute(Operation *op,
unsigned regionIdx,
unsigned resIdx,
NamedAttribute resAttr) {
auto funcOp = dyn_cast<FunctionOpInterface>(op);
if (!funcOp)
return success();
Type resType = funcOp.getResultTypes()[resIdx];
// Check to see if this function has a void return with a result attribute
// to it. It isn't clear what semantics we would assign to that.
if (llvm::isa<LLVMVoidType>(resType))
return op->emitError() << "cannot attach result attributes to functions "
"with a void return";
// Check to see if this attribute is allowed as a result attribute. Only
// explicitly forbidden LLVM attributes will cause an error.
auto name = resAttr.getName();
if (name == LLVMDialect::getAllocAlignAttrName() ||
name == LLVMDialect::getAllocatedPointerAttrName() ||
name == LLVMDialect::getByValAttrName() ||
name == LLVMDialect::getByRefAttrName() ||
name == LLVMDialect::getInAllocaAttrName() ||
name == LLVMDialect::getNestAttrName() ||
name == LLVMDialect::getNoCaptureAttrName() ||
name == LLVMDialect::getNoFreeAttrName() ||
name == LLVMDialect::getPreallocatedAttrName() ||
name == LLVMDialect::getReadnoneAttrName() ||
name == LLVMDialect::getReadonlyAttrName() ||
name == LLVMDialect::getReturnedAttrName() ||
name == LLVMDialect::getStackAlignmentAttrName() ||
name == LLVMDialect::getStructRetAttrName() ||
name == LLVMDialect::getWriteOnlyAttrName())
return op->emitError() << name << " is not a valid result attribute";
return verifyParameterAttribute(op, resType, resAttr);
}
Operation *LLVMDialect::materializeConstant(OpBuilder &builder, Attribute value,
Type type, Location loc) {
// If this was folded from an operation other than llvm.mlir.constant, it
// should be materialized as such. Note that an llvm.mlir.zero may fold into
// a builtin zero attribute and thus will materialize as a llvm.mlir.constant.
if (auto symbol = dyn_cast<FlatSymbolRefAttr>(value))
if (isa<LLVM::LLVMPointerType>(type))
return LLVM::AddressOfOp::create(builder, loc, type, symbol);
if (isa<LLVM::UndefAttr>(value))
return LLVM::UndefOp::create(builder, loc, type);
if (isa<LLVM::PoisonAttr>(value))
return LLVM::PoisonOp::create(builder, loc, type);
if (isa<LLVM::ZeroAttr>(value))
return LLVM::ZeroOp::create(builder, loc, type);
// Otherwise try materializing it as a regular llvm.mlir.constant op.
return LLVM::ConstantOp::materialize(builder, value, type, loc);
}
//===----------------------------------------------------------------------===//
// Utility functions.
//===----------------------------------------------------------------------===//
Value mlir::LLVM::createGlobalString(Location loc, OpBuilder &builder,
StringRef name, StringRef value,
LLVM::Linkage linkage) {
assert(builder.getInsertionBlock() &&
builder.getInsertionBlock()->getParentOp() &&
"expected builder to point to a block constrained in an op");
auto module =
builder.getInsertionBlock()->getParentOp()->getParentOfType<ModuleOp>();
assert(module && "builder points to an op outside of a module");
// Create the global at the entry of the module.
OpBuilder moduleBuilder(module.getBodyRegion(), builder.getListener());
MLIRContext *ctx = builder.getContext();
auto type = LLVM::LLVMArrayType::get(IntegerType::get(ctx, 8), value.size());
auto global = LLVM::GlobalOp::create(
moduleBuilder, loc, type, /*isConstant=*/true, linkage, name,
builder.getStringAttr(value), /*alignment=*/0);
LLVMPointerType ptrType = LLVMPointerType::get(ctx);
// Get the pointer to the first character in the global string.
Value globalPtr =
LLVM::AddressOfOp::create(builder, loc, ptrType, global.getSymNameAttr());
return LLVM::GEPOp::create(builder, loc, ptrType, type, globalPtr,
ArrayRef<GEPArg>{0, 0});
}
bool mlir::LLVM::satisfiesLLVMModule(Operation *op) {
return op->hasTrait<OpTrait::SymbolTable>() &&
op->hasTrait<OpTrait::IsIsolatedFromAbove>();
}
Operation *mlir::LLVM::parentLLVMModule(Operation *op) {
Operation *module = op->getParentOp();
while (module && !satisfiesLLVMModule(module))
module = module->getParentOp();
assert(module && "unexpected operation outside of a module");
return module;
}
Reference in New Issue View Git Blame Copy Permalink