shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/clang/lib/AST/ByteCode/InterpBuiltin.cpp
Islam Imad 246528cb3a
[clang][bytecode] Unify elementwise integer builtins using callback pattern (#169957) 
This patch refactors the handling of elementwise integer unary
operations to use a unified callback-based approach, eliminating code
duplication.

Changes:
- Extended interp__builtin_elementwise_int_unaryop to handle vector types
- Replaced BI__builtin_elementwise_popcount with callback invocation
- Replaced BI__builtin_elementwise_bitreverse with callback invocation
- Removed  interp__builtin_elementwise_popcount function

The new approach uses a lambda function to specify the operation
(popcount or reverseBits), which is applied uniformly to both scalar and
vector operands. This reduces code duplication and makes it easier to
add similar builtins in the future.

Fixes #169657
2025-11-29 17:38:17 +00:00

5523 lines
198 KiB
C++
Raw Blame History

//===--- InterpBuiltin.cpp - Interpreter for the constexpr VM ---*- C++ -*-===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
#include "../ExprConstShared.h"
#include "Boolean.h"
#include "EvalEmitter.h"
#include "InterpBuiltinBitCast.h"
#include "InterpHelpers.h"
#include "PrimType.h"
#include "Program.h"
#include "clang/AST/InferAlloc.h"
#include "clang/AST/OSLog.h"
#include "clang/AST/RecordLayout.h"
#include "clang/Basic/Builtins.h"
#include "clang/Basic/TargetBuiltins.h"
#include "clang/Basic/TargetInfo.h"
#include "llvm/ADT/StringExtras.h"
#include "llvm/Support/AllocToken.h"
#include "llvm/Support/ErrorHandling.h"
#include "llvm/Support/SipHash.h"
namespace clang {
namespace interp {
[[maybe_unused]] static bool isNoopBuiltin(unsigned ID) {
switch (ID) {
case Builtin::BIas_const:
case Builtin::BIforward:
case Builtin::BIforward_like:
case Builtin::BImove:
case Builtin::BImove_if_noexcept:
case Builtin::BIaddressof:
case Builtin::BI__addressof:
case Builtin::BI__builtin_addressof:
case Builtin::BI__builtin_launder:
return true;
default:
return false;
}
return false;
}
static void discard(InterpStack &Stk, PrimType T) {
TYPE_SWITCH(T, { Stk.discard<T>(); });
}
static APSInt popToAPSInt(InterpStack &Stk, PrimType T) {
INT_TYPE_SWITCH(T, return Stk.pop<T>().toAPSInt());
}
static APSInt popToAPSInt(InterpState &S, const Expr *E) {
return popToAPSInt(S.Stk, *S.getContext().classify(E->getType()));
}
static APSInt popToAPSInt(InterpState &S, QualType T) {
return popToAPSInt(S.Stk, *S.getContext().classify(T));
}
/// Pushes \p Val on the stack as the type given by \p QT.
static void pushInteger(InterpState &S, const APSInt &Val, QualType QT) {
assert(QT->isSignedIntegerOrEnumerationType() ||
QT->isUnsignedIntegerOrEnumerationType());
OptPrimType T = S.getContext().classify(QT);
assert(T);
unsigned BitWidth = S.getASTContext().getTypeSize(QT);
if (T == PT_IntAPS) {
auto Result = S.allocAP<IntegralAP<true>>(BitWidth);
Result.copy(Val);
S.Stk.push<IntegralAP<true>>(Result);
return;
}
if (T == PT_IntAP) {
auto Result = S.allocAP<IntegralAP<false>>(BitWidth);
Result.copy(Val);
S.Stk.push<IntegralAP<false>>(Result);
return;
}
if (QT->isSignedIntegerOrEnumerationType()) {
int64_t V = Val.getSExtValue();
INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
} else {
assert(QT->isUnsignedIntegerOrEnumerationType());
uint64_t V = Val.getZExtValue();
INT_TYPE_SWITCH(*T, { S.Stk.push<T>(T::from(V, BitWidth)); });
}
}
template <typename T>
static void pushInteger(InterpState &S, T Val, QualType QT) {
if constexpr (std::is_same_v<T, APInt>)
pushInteger(S, APSInt(Val, !std::is_signed_v<T>), QT);
else if constexpr (std::is_same_v<T, APSInt>)
pushInteger(S, Val, QT);
else
pushInteger(S,
APSInt(APInt(sizeof(T) * 8, static_cast<uint64_t>(Val),
std::is_signed_v<T>),
!std::is_signed_v<T>),
QT);
}
static void assignInteger(InterpState &S, const Pointer &Dest, PrimType ValueT,
const APSInt &Value) {
if (ValueT == PT_IntAPS) {
Dest.deref<IntegralAP<true>>() =
S.allocAP<IntegralAP<true>>(Value.getBitWidth());
Dest.deref<IntegralAP<true>>().copy(Value);
} else if (ValueT == PT_IntAP) {
Dest.deref<IntegralAP<false>>() =
S.allocAP<IntegralAP<false>>(Value.getBitWidth());
Dest.deref<IntegralAP<false>>().copy(Value);
} else {
INT_TYPE_SWITCH_NO_BOOL(
ValueT, { Dest.deref<T>() = T::from(static_cast<T>(Value)); });
}
}
static QualType getElemType(const Pointer &P) {
const Descriptor *Desc = P.getFieldDesc();
QualType T = Desc->getType();
if (Desc->isPrimitive())
return T;
if (T->isPointerType())
return T->getAs<PointerType>()->getPointeeType();
if (Desc->isArray())
return Desc->getElemQualType();
if (const auto *AT = T->getAsArrayTypeUnsafe())
return AT->getElementType();
return T;
}
static void diagnoseNonConstexprBuiltin(InterpState &S, CodePtr OpPC,
unsigned ID) {
if (!S.diagnosing())
return;
auto Loc = S.Current->getSource(OpPC);
if (S.getLangOpts().CPlusPlus11)
S.CCEDiag(Loc, diag::note_constexpr_invalid_function)
<< /*isConstexpr=*/0 << /*isConstructor=*/0
<< S.getASTContext().BuiltinInfo.getQuotedName(ID);
else
S.CCEDiag(Loc, diag::note_invalid_subexpr_in_const_expr);
}
static llvm::APSInt convertBoolVectorToInt(const Pointer &Val) {
assert(Val.getFieldDesc()->isPrimitiveArray() &&
Val.getFieldDesc()->getElemQualType()->isBooleanType() &&
"Not a boolean vector");
unsigned NumElems = Val.getNumElems();
// Each element is one bit, so create an integer with NumElts bits.
llvm::APSInt Result(NumElems, 0);
for (unsigned I = 0; I != NumElems; ++I) {
if (Val.elem<bool>(I))
Result.setBit(I);
}
return Result;
}
static bool interp__builtin_is_constant_evaluated(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
unsigned Depth = S.Current->getDepth();
auto isStdCall = [](const FunctionDecl *F) -> bool {
return F && F->isInStdNamespace() && F->getIdentifier() &&
F->getIdentifier()->isStr("is_constant_evaluated");
};
const InterpFrame *Caller = Frame->Caller;
// The current frame is the one for __builtin_is_constant_evaluated.
// The one above that, potentially the one for std::is_constant_evaluated().
if (S.inConstantContext() && !S.checkingPotentialConstantExpression() &&
S.getEvalStatus().Diag &&
(Depth == 0 || (Depth == 1 && isStdCall(Frame->getCallee())))) {
if (Caller && isStdCall(Frame->getCallee())) {
const Expr *E = Caller->getExpr(Caller->getRetPC());
S.report(E->getExprLoc(),
diag::warn_is_constant_evaluated_always_true_constexpr)
<< "std::is_constant_evaluated" << E->getSourceRange();
} else {
S.report(Call->getExprLoc(),
diag::warn_is_constant_evaluated_always_true_constexpr)
<< "__builtin_is_constant_evaluated" << Call->getSourceRange();
}
}
S.Stk.push<Boolean>(Boolean::from(S.inConstantContext()));
return true;
}
// __builtin_assume(int)
static bool interp__builtin_assume(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
assert(Call->getNumArgs() == 1);
discard(S.Stk, *S.getContext().classify(Call->getArg(0)));
return true;
}
static bool interp__builtin_strcmp(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call, unsigned ID) {
uint64_t Limit = ~static_cast<uint64_t>(0);
if (ID == Builtin::BIstrncmp || ID == Builtin::BI__builtin_strncmp ||
ID == Builtin::BIwcsncmp || ID == Builtin::BI__builtin_wcsncmp)
Limit = popToAPSInt(S.Stk, *S.getContext().classify(Call->getArg(2)))
.getZExtValue();
const Pointer &B = S.Stk.pop<Pointer>();
const Pointer &A = S.Stk.pop<Pointer>();
if (ID == Builtin::BIstrcmp || ID == Builtin::BIstrncmp ||
ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp)
diagnoseNonConstexprBuiltin(S, OpPC, ID);
if (Limit == 0) {
pushInteger(S, 0, Call->getType());
return true;
}
if (!CheckLive(S, OpPC, A, AK_Read) || !CheckLive(S, OpPC, B, AK_Read))
return false;
if (A.isDummy() || B.isDummy())
return false;
if (!A.isBlockPointer() || !B.isBlockPointer())
return false;
bool IsWide = ID == Builtin::BIwcscmp || ID == Builtin::BIwcsncmp ||
ID == Builtin::BI__builtin_wcscmp ||
ID == Builtin::BI__builtin_wcsncmp;
assert(A.getFieldDesc()->isPrimitiveArray());
assert(B.getFieldDesc()->isPrimitiveArray());
// Different element types shouldn't happen, but with casts they can.
if (!S.getASTContext().hasSameUnqualifiedType(getElemType(A), getElemType(B)))
return false;
PrimType ElemT = *S.getContext().classify(getElemType(A));
auto returnResult = [&](int V) -> bool {
pushInteger(S, V, Call->getType());
return true;
};
unsigned IndexA = A.getIndex();
unsigned IndexB = B.getIndex();
uint64_t Steps = 0;
for (;; ++IndexA, ++IndexB, ++Steps) {
if (Steps >= Limit)
break;
const Pointer &PA = A.atIndex(IndexA);
const Pointer &PB = B.atIndex(IndexB);
if (!CheckRange(S, OpPC, PA, AK_Read) ||
!CheckRange(S, OpPC, PB, AK_Read)) {
return false;
}
if (IsWide) {
INT_TYPE_SWITCH(ElemT, {
T CA = PA.deref<T>();
T CB = PB.deref<T>();
if (CA > CB)
return returnResult(1);
if (CA < CB)
return returnResult(-1);
if (CA.isZero() || CB.isZero())
return returnResult(0);
});
continue;
}
uint8_t CA = PA.deref<uint8_t>();
uint8_t CB = PB.deref<uint8_t>();
if (CA > CB)
return returnResult(1);
if (CA < CB)
return returnResult(-1);
if (CA == 0 || CB == 0)
return returnResult(0);
}
return returnResult(0);
}
static bool interp__builtin_strlen(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call, unsigned ID) {
const Pointer &StrPtr = S.Stk.pop<Pointer>().expand();
if (ID == Builtin::BIstrlen || ID == Builtin::BIwcslen)
diagnoseNonConstexprBuiltin(S, OpPC, ID);
if (!CheckArray(S, OpPC, StrPtr))
return false;
if (!CheckLive(S, OpPC, StrPtr, AK_Read))
return false;
if (!CheckDummy(S, OpPC, StrPtr.block(), AK_Read))
return false;
if (!StrPtr.getFieldDesc()->isPrimitiveArray())
return false;
assert(StrPtr.getFieldDesc()->isPrimitiveArray());
unsigned ElemSize = StrPtr.getFieldDesc()->getElemSize();
if (ID == Builtin::BI__builtin_wcslen || ID == Builtin::BIwcslen) {
[[maybe_unused]] const ASTContext &AC = S.getASTContext();
assert(ElemSize == AC.getTypeSizeInChars(AC.getWCharType()).getQuantity());
}
size_t Len = 0;
for (size_t I = StrPtr.getIndex();; ++I, ++Len) {
const Pointer &ElemPtr = StrPtr.atIndex(I);
if (!CheckRange(S, OpPC, ElemPtr, AK_Read))
return false;
uint32_t Val;
switch (ElemSize) {
case 1:
Val = ElemPtr.deref<uint8_t>();
break;
case 2:
Val = ElemPtr.deref<uint16_t>();
break;
case 4:
Val = ElemPtr.deref<uint32_t>();
break;
default:
llvm_unreachable("Unsupported char size");
}
if (Val == 0)
break;
}
pushInteger(S, Len, Call->getType());
return true;
}
static bool interp__builtin_nan(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame, const CallExpr *Call,
bool Signaling) {
const Pointer &Arg = S.Stk.pop<Pointer>();
if (!CheckLoad(S, OpPC, Arg))
return false;
assert(Arg.getFieldDesc()->isPrimitiveArray());
// Convert the given string to an integer using StringRef's API.
llvm::APInt Fill;
std::string Str;
assert(Arg.getNumElems() >= 1);
for (unsigned I = 0;; ++I) {
const Pointer &Elem = Arg.atIndex(I);
if (!CheckLoad(S, OpPC, Elem))
return false;
if (Elem.deref<int8_t>() == 0)
break;
Str += Elem.deref<char>();
}
// Treat empty strings as if they were zero.
if (Str.empty())
Fill = llvm::APInt(32, 0);
else if (StringRef(Str).getAsInteger(0, Fill))
return false;
const llvm::fltSemantics &TargetSemantics =
S.getASTContext().getFloatTypeSemantics(
Call->getDirectCallee()->getReturnType());
Floating Result = S.allocFloat(TargetSemantics);
if (S.getASTContext().getTargetInfo().isNan2008()) {
if (Signaling)
Result.copy(
llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
else
Result.copy(
llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
} else {
// Prior to IEEE 754-2008, architectures were allowed to choose whether
// the first bit of their significand was set for qNaN or sNaN. MIPS chose
// a different encoding to what became a standard in 2008, and for pre-
// 2008 revisions, MIPS interpreted sNaN-2008 as qNan and qNaN-2008 as
// sNaN. This is now known as "legacy NaN" encoding.
if (Signaling)
Result.copy(
llvm::APFloat::getQNaN(TargetSemantics, /*Negative=*/false, &Fill));
else
Result.copy(
llvm::APFloat::getSNaN(TargetSemantics, /*Negative=*/false, &Fill));
}
S.Stk.push<Floating>(Result);
return true;
}
static bool interp__builtin_inf(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const llvm::fltSemantics &TargetSemantics =
S.getASTContext().getFloatTypeSemantics(
Call->getDirectCallee()->getReturnType());
Floating Result = S.allocFloat(TargetSemantics);
Result.copy(APFloat::getInf(TargetSemantics));
S.Stk.push<Floating>(Result);
return true;
}
static bool interp__builtin_copysign(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame) {
const Floating &Arg2 = S.Stk.pop<Floating>();
const Floating &Arg1 = S.Stk.pop<Floating>();
Floating Result = S.allocFloat(Arg1.getSemantics());
APFloat Copy = Arg1.getAPFloat();
Copy.copySign(Arg2.getAPFloat());
Result.copy(Copy);
S.Stk.push<Floating>(Result);
return true;
}
static bool interp__builtin_fmin(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame, bool IsNumBuiltin) {
const Floating &RHS = S.Stk.pop<Floating>();
const Floating &LHS = S.Stk.pop<Floating>();
Floating Result = S.allocFloat(LHS.getSemantics());
if (IsNumBuiltin)
Result.copy(llvm::minimumnum(LHS.getAPFloat(), RHS.getAPFloat()));
else
Result.copy(minnum(LHS.getAPFloat(), RHS.getAPFloat()));
S.Stk.push<Floating>(Result);
return true;
}
static bool interp__builtin_fmax(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame, bool IsNumBuiltin) {
const Floating &RHS = S.Stk.pop<Floating>();
const Floating &LHS = S.Stk.pop<Floating>();
Floating Result = S.allocFloat(LHS.getSemantics());
if (IsNumBuiltin)
Result.copy(llvm::maximumnum(LHS.getAPFloat(), RHS.getAPFloat()));
else
Result.copy(maxnum(LHS.getAPFloat(), RHS.getAPFloat()));
S.Stk.push<Floating>(Result);
return true;
}
/// Defined as __builtin_isnan(...), to accommodate the fact that it can
/// take a float, double, long double, etc.
/// But for us, that's all a Floating anyway.
static bool interp__builtin_isnan(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg = S.Stk.pop<Floating>();
pushInteger(S, Arg.isNan(), Call->getType());
return true;
}
static bool interp__builtin_issignaling(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg = S.Stk.pop<Floating>();
pushInteger(S, Arg.isSignaling(), Call->getType());
return true;
}
static bool interp__builtin_isinf(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame, bool CheckSign,
const CallExpr *Call) {
const Floating &Arg = S.Stk.pop<Floating>();
APFloat F = Arg.getAPFloat();
bool IsInf = F.isInfinity();
if (CheckSign)
pushInteger(S, IsInf ? (F.isNegative() ? -1 : 1) : 0, Call->getType());
else
pushInteger(S, IsInf, Call->getType());
return true;
}
static bool interp__builtin_isfinite(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg = S.Stk.pop<Floating>();
pushInteger(S, Arg.isFinite(), Call->getType());
return true;
}
static bool interp__builtin_isnormal(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg = S.Stk.pop<Floating>();
pushInteger(S, Arg.isNormal(), Call->getType());
return true;
}
static bool interp__builtin_issubnormal(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg = S.Stk.pop<Floating>();
pushInteger(S, Arg.isDenormal(), Call->getType());
return true;
}
static bool interp__builtin_iszero(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg = S.Stk.pop<Floating>();
pushInteger(S, Arg.isZero(), Call->getType());
return true;
}
static bool interp__builtin_signbit(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg = S.Stk.pop<Floating>();
pushInteger(S, Arg.isNegative(), Call->getType());
return true;
}
static bool interp_floating_comparison(InterpState &S, CodePtr OpPC,
const CallExpr *Call, unsigned ID) {
const Floating &RHS = S.Stk.pop<Floating>();
const Floating &LHS = S.Stk.pop<Floating>();
pushInteger(
S,
[&] {
switch (ID) {
case Builtin::BI__builtin_isgreater:
return LHS > RHS;
case Builtin::BI__builtin_isgreaterequal:
return LHS >= RHS;
case Builtin::BI__builtin_isless:
return LHS < RHS;
case Builtin::BI__builtin_islessequal:
return LHS <= RHS;
case Builtin::BI__builtin_islessgreater: {
ComparisonCategoryResult Cmp = LHS.compare(RHS);
return Cmp == ComparisonCategoryResult::Less ||
Cmp == ComparisonCategoryResult::Greater;
}
case Builtin::BI__builtin_isunordered:
return LHS.compare(RHS) == ComparisonCategoryResult::Unordered;
default:
llvm_unreachable("Unexpected builtin ID: Should be a floating point "
"comparison function");
}
}(),
Call->getType());
return true;
}
/// First parameter to __builtin_isfpclass is the floating value, the
/// second one is an integral value.
static bool interp__builtin_isfpclass(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
APSInt FPClassArg = popToAPSInt(S, Call->getArg(1));
const Floating &F = S.Stk.pop<Floating>();
int32_t Result = static_cast<int32_t>(
(F.classify() & std::move(FPClassArg)).getZExtValue());
pushInteger(S, Result, Call->getType());
return true;
}
/// Five int values followed by one floating value.
/// __builtin_fpclassify(int, int, int, int, int, float)
static bool interp__builtin_fpclassify(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Val = S.Stk.pop<Floating>();
PrimType IntT = *S.getContext().classify(Call->getArg(0));
APSInt Values[5];
for (unsigned I = 0; I != 5; ++I)
Values[4 - I] = popToAPSInt(S.Stk, IntT);
unsigned Index;
switch (Val.getCategory()) {
case APFloat::fcNaN:
Index = 0;
break;
case APFloat::fcInfinity:
Index = 1;
break;
case APFloat::fcNormal:
Index = Val.isDenormal() ? 3 : 2;
break;
case APFloat::fcZero:
Index = 4;
break;
}
// The last argument is first on the stack.
assert(Index <= 4);
pushInteger(S, Values[Index], Call->getType());
return true;
}
static inline Floating abs(InterpState &S, const Floating &In) {
if (!In.isNegative())
return In;
Floating Output = S.allocFloat(In.getSemantics());
APFloat New = In.getAPFloat();
New.changeSign();
Output.copy(New);
return Output;
}
// The C standard says "fabs raises no floating-point exceptions,
// even if x is a signaling NaN. The returned value is independent of
// the current rounding direction mode." Therefore constant folding can
// proceed without regard to the floating point settings.
// Reference, WG14 N2478 F.10.4.3
static bool interp__builtin_fabs(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame) {
const Floating &Val = S.Stk.pop<Floating>();
S.Stk.push<Floating>(abs(S, Val));
return true;
}
static bool interp__builtin_abs(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
APSInt Val = popToAPSInt(S, Call->getArg(0));
if (Val ==
APSInt(APInt::getSignedMinValue(Val.getBitWidth()), /*IsUnsigned=*/false))
return false;
if (Val.isNegative())
Val.negate();
pushInteger(S, Val, Call->getType());
return true;
}
static bool interp__builtin_popcount(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
APSInt Val;
if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
const Pointer &Arg = S.Stk.pop<Pointer>();
Val = convertBoolVectorToInt(Arg);
} else {
Val = popToAPSInt(S, Call->getArg(0));
}
pushInteger(S, Val.popcount(), Call->getType());
return true;
}
static bool interp__builtin_classify_type(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
// This is an unevaluated call, so there are no arguments on the stack.
assert(Call->getNumArgs() == 1);
const Expr *Arg = Call->getArg(0);
GCCTypeClass ResultClass =
EvaluateBuiltinClassifyType(Arg->getType(), S.getLangOpts());
int32_t ReturnVal = static_cast<int32_t>(ResultClass);
pushInteger(S, ReturnVal, Call->getType());
return true;
}
// __builtin_expect(long, long)
// __builtin_expect_with_probability(long, long, double)
static bool interp__builtin_expect(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
// The return value is simply the value of the first parameter.
// We ignore the probability.
unsigned NumArgs = Call->getNumArgs();
assert(NumArgs == 2 || NumArgs == 3);
PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
if (NumArgs == 3)
S.Stk.discard<Floating>();
discard(S.Stk, ArgT);
APSInt Val = popToAPSInt(S.Stk, ArgT);
pushInteger(S, Val, Call->getType());
return true;
}
static bool interp__builtin_addressof(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
#ifndef NDEBUG
assert(Call->getArg(0)->isLValue());
PrimType PtrT = S.getContext().classify(Call->getArg(0)).value_or(PT_Ptr);
assert(PtrT == PT_Ptr &&
"Unsupported pointer type passed to __builtin_addressof()");
#endif
return true;
}
static bool interp__builtin_move(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
return Call->getDirectCallee()->isConstexpr();
}
static bool interp__builtin_eh_return_data_regno(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
APSInt Arg = popToAPSInt(S, Call->getArg(0));
int Result = S.getASTContext().getTargetInfo().getEHDataRegisterNumber(
Arg.getZExtValue());
pushInteger(S, Result, Call->getType());
return true;
}
// Two integral values followed by a pointer (lhs, rhs, resultOut)
static bool interp__builtin_overflowop(InterpState &S, CodePtr OpPC,
const CallExpr *Call,
unsigned BuiltinOp) {
const Pointer &ResultPtr = S.Stk.pop<Pointer>();
if (ResultPtr.isDummy() || !ResultPtr.isBlockPointer())
return false;
PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());
PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());
APSInt RHS = popToAPSInt(S.Stk, RHST);
APSInt LHS = popToAPSInt(S.Stk, LHST);
QualType ResultType = Call->getArg(2)->getType()->getPointeeType();
PrimType ResultT = *S.getContext().classify(ResultType);
bool Overflow;
APSInt Result;
if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
BuiltinOp == Builtin::BI__builtin_sub_overflow ||
BuiltinOp == Builtin::BI__builtin_mul_overflow) {
bool IsSigned = LHS.isSigned() || RHS.isSigned() ||
ResultType->isSignedIntegerOrEnumerationType();
bool AllSigned = LHS.isSigned() && RHS.isSigned() &&
ResultType->isSignedIntegerOrEnumerationType();
uint64_t LHSSize = LHS.getBitWidth();
uint64_t RHSSize = RHS.getBitWidth();
uint64_t ResultSize = S.getASTContext().getTypeSize(ResultType);
uint64_t MaxBits = std::max(std::max(LHSSize, RHSSize), ResultSize);
// Add an additional bit if the signedness isn't uniformly agreed to. We
// could do this ONLY if there is a signed and an unsigned that both have
// MaxBits, but the code to check that is pretty nasty. The issue will be
// caught in the shrink-to-result later anyway.
if (IsSigned && !AllSigned)
++MaxBits;
LHS = APSInt(LHS.extOrTrunc(MaxBits), !IsSigned);
RHS = APSInt(RHS.extOrTrunc(MaxBits), !IsSigned);
Result = APSInt(MaxBits, !IsSigned);
}
// Find largest int.
switch (BuiltinOp) {
default:
llvm_unreachable("Invalid value for BuiltinOp");
case Builtin::BI__builtin_add_overflow:
case Builtin::BI__builtin_sadd_overflow:
case Builtin::BI__builtin_saddl_overflow:
case Builtin::BI__builtin_saddll_overflow:
case Builtin::BI__builtin_uadd_overflow:
case Builtin::BI__builtin_uaddl_overflow:
case Builtin::BI__builtin_uaddll_overflow:
Result = LHS.isSigned() ? LHS.sadd_ov(RHS, Overflow)
: LHS.uadd_ov(RHS, Overflow);
break;
case Builtin::BI__builtin_sub_overflow:
case Builtin::BI__builtin_ssub_overflow:
case Builtin::BI__builtin_ssubl_overflow:
case Builtin::BI__builtin_ssubll_overflow:
case Builtin::BI__builtin_usub_overflow:
case Builtin::BI__builtin_usubl_overflow:
case Builtin::BI__builtin_usubll_overflow:
Result = LHS.isSigned() ? LHS.ssub_ov(RHS, Overflow)
: LHS.usub_ov(RHS, Overflow);
break;
case Builtin::BI__builtin_mul_overflow:
case Builtin::BI__builtin_smul_overflow:
case Builtin::BI__builtin_smull_overflow:
case Builtin::BI__builtin_smulll_overflow:
case Builtin::BI__builtin_umul_overflow:
case Builtin::BI__builtin_umull_overflow:
case Builtin::BI__builtin_umulll_overflow:
Result = LHS.isSigned() ? LHS.smul_ov(RHS, Overflow)
: LHS.umul_ov(RHS, Overflow);
break;
}
// In the case where multiple sizes are allowed, truncate and see if
// the values are the same.
if (BuiltinOp == Builtin::BI__builtin_add_overflow ||
BuiltinOp == Builtin::BI__builtin_sub_overflow ||
BuiltinOp == Builtin::BI__builtin_mul_overflow) {
// APSInt doesn't have a TruncOrSelf, so we use extOrTrunc instead,
// since it will give us the behavior of a TruncOrSelf in the case where
// its parameter <= its size. We previously set Result to be at least the
// type-size of the result, so getTypeSize(ResultType) <= Resu
APSInt Temp = Result.extOrTrunc(S.getASTContext().getTypeSize(ResultType));
Temp.setIsSigned(ResultType->isSignedIntegerOrEnumerationType());
if (!APSInt::isSameValue(Temp, Result))
Overflow = true;
Result = std::move(Temp);
}
// Write Result to ResultPtr and put Overflow on the stack.
assignInteger(S, ResultPtr, ResultT, Result);
if (ResultPtr.canBeInitialized())
ResultPtr.initialize();
assert(Call->getDirectCallee()->getReturnType()->isBooleanType());
S.Stk.push<Boolean>(Overflow);
return true;
}
/// Three integral values followed by a pointer (lhs, rhs, carry, carryOut).
static bool interp__builtin_carryop(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call, unsigned BuiltinOp) {
const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
PrimType LHST = *S.getContext().classify(Call->getArg(0)->getType());
PrimType RHST = *S.getContext().classify(Call->getArg(1)->getType());
APSInt CarryIn = popToAPSInt(S.Stk, LHST);
APSInt RHS = popToAPSInt(S.Stk, RHST);
APSInt LHS = popToAPSInt(S.Stk, LHST);
if (CarryOutPtr.isDummy() || !CarryOutPtr.isBlockPointer())
return false;
APSInt CarryOut;
APSInt Result;
// Copy the number of bits and sign.
Result = LHS;
CarryOut = LHS;
bool FirstOverflowed = false;
bool SecondOverflowed = false;
switch (BuiltinOp) {
default:
llvm_unreachable("Invalid value for BuiltinOp");
case Builtin::BI__builtin_addcb:
case Builtin::BI__builtin_addcs:
case Builtin::BI__builtin_addc:
case Builtin::BI__builtin_addcl:
case Builtin::BI__builtin_addcll:
Result =
LHS.uadd_ov(RHS, FirstOverflowed).uadd_ov(CarryIn, SecondOverflowed);
break;
case Builtin::BI__builtin_subcb:
case Builtin::BI__builtin_subcs:
case Builtin::BI__builtin_subc:
case Builtin::BI__builtin_subcl:
case Builtin::BI__builtin_subcll:
Result =
LHS.usub_ov(RHS, FirstOverflowed).usub_ov(CarryIn, SecondOverflowed);
break;
}
// It is possible for both overflows to happen but CGBuiltin uses an OR so
// this is consistent.
CarryOut = (uint64_t)(FirstOverflowed | SecondOverflowed);
QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
PrimType CarryOutT = *S.getContext().classify(CarryOutType);
assignInteger(S, CarryOutPtr, CarryOutT, CarryOut);
CarryOutPtr.initialize();
assert(Call->getType() == Call->getArg(0)->getType());
pushInteger(S, Result, Call->getType());
return true;
}
static bool interp__builtin_clz(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame, const CallExpr *Call,
unsigned BuiltinOp) {
std::optional<APSInt> Fallback;
if (BuiltinOp == Builtin::BI__builtin_clzg && Call->getNumArgs() == 2)
Fallback = popToAPSInt(S, Call->getArg(1));
APSInt Val;
if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
const Pointer &Arg = S.Stk.pop<Pointer>();
Val = convertBoolVectorToInt(Arg);
} else {
Val = popToAPSInt(S, Call->getArg(0));
}
// When the argument is 0, the result of GCC builtins is undefined, whereas
// for Microsoft intrinsics, the result is the bit-width of the argument.
bool ZeroIsUndefined = BuiltinOp != Builtin::BI__lzcnt16 &&
BuiltinOp != Builtin::BI__lzcnt &&
BuiltinOp != Builtin::BI__lzcnt64;
if (Val == 0) {
if (Fallback) {
pushInteger(S, *Fallback, Call->getType());
return true;
}
if (ZeroIsUndefined)
return false;
}
pushInteger(S, Val.countl_zero(), Call->getType());
return true;
}
static bool interp__builtin_ctz(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame, const CallExpr *Call,
unsigned BuiltinID) {
std::optional<APSInt> Fallback;
if (BuiltinID == Builtin::BI__builtin_ctzg && Call->getNumArgs() == 2)
Fallback = popToAPSInt(S, Call->getArg(1));
APSInt Val;
if (Call->getArg(0)->getType()->isExtVectorBoolType()) {
const Pointer &Arg = S.Stk.pop<Pointer>();
Val = convertBoolVectorToInt(Arg);
} else {
Val = popToAPSInt(S, Call->getArg(0));
}
if (Val == 0) {
if (Fallback) {
pushInteger(S, *Fallback, Call->getType());
return true;
}
return false;
}
pushInteger(S, Val.countr_zero(), Call->getType());
return true;
}
static bool interp__builtin_bswap(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const APSInt &Val = popToAPSInt(S, Call->getArg(0));
if (Val.getBitWidth() == 8)
pushInteger(S, Val, Call->getType());
else
pushInteger(S, Val.byteSwap(), Call->getType());
return true;
}
/// bool __atomic_always_lock_free(size_t, void const volatile*)
/// bool __atomic_is_lock_free(size_t, void const volatile*)
static bool interp__builtin_atomic_lock_free(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call,
unsigned BuiltinOp) {
auto returnBool = [&S](bool Value) -> bool {
S.Stk.push<Boolean>(Value);
return true;
};
const Pointer &Ptr = S.Stk.pop<Pointer>();
const APSInt &SizeVal = popToAPSInt(S, Call->getArg(0));
// For __atomic_is_lock_free(sizeof(_Atomic(T))), if the size is a power
// of two less than or equal to the maximum inline atomic width, we know it
// is lock-free. If the size isn't a power of two, or greater than the
// maximum alignment where we promote atomics, we know it is not lock-free
// (at least not in the sense of atomic_is_lock_free). Otherwise,
// the answer can only be determined at runtime; for example, 16-byte
// atomics have lock-free implementations on some, but not all,
// x86-64 processors.
// Check power-of-two.
CharUnits Size = CharUnits::fromQuantity(SizeVal.getZExtValue());
if (Size.isPowerOfTwo()) {
// Check against inlining width.
unsigned InlineWidthBits =
S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits)) {
// OK, we will inline appropriately-aligned operations of this size,
// and _Atomic(T) is appropriately-aligned.
if (Size == CharUnits::One())
return returnBool(true);
// Same for null pointers.
assert(BuiltinOp != Builtin::BI__c11_atomic_is_lock_free);
if (Ptr.isZero())
return returnBool(true);
if (Ptr.isIntegralPointer()) {
uint64_t IntVal = Ptr.getIntegerRepresentation();
if (APSInt(APInt(64, IntVal, false), true).isAligned(Size.getAsAlign()))
return returnBool(true);
}
const Expr *PtrArg = Call->getArg(1);
// Otherwise, check if the type's alignment against Size.
if (const auto *ICE = dyn_cast<ImplicitCastExpr>(PtrArg)) {
// Drop the potential implicit-cast to 'const volatile void*', getting
// the underlying type.
if (ICE->getCastKind() == CK_BitCast)
PtrArg = ICE->getSubExpr();
}
if (const auto *PtrTy = PtrArg->getType()->getAs<PointerType>()) {
QualType PointeeType = PtrTy->getPointeeType();
if (!PointeeType->isIncompleteType() &&
S.getASTContext().getTypeAlignInChars(PointeeType) >= Size) {
// OK, we will inline operations on this object.
return returnBool(true);
}
}
}
}
if (BuiltinOp == Builtin::BI__atomic_always_lock_free)
return returnBool(false);
return false;
}
/// bool __c11_atomic_is_lock_free(size_t)
static bool interp__builtin_c11_atomic_is_lock_free(InterpState &S,
CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const APSInt &SizeVal = popToAPSInt(S, Call->getArg(0));
CharUnits Size = CharUnits::fromQuantity(SizeVal.getZExtValue());
if (Size.isPowerOfTwo()) {
// Check against inlining width.
unsigned InlineWidthBits =
S.getASTContext().getTargetInfo().getMaxAtomicInlineWidth();
if (Size <= S.getASTContext().toCharUnitsFromBits(InlineWidthBits)) {
S.Stk.push<Boolean>(true);
return true;
}
}
return false; // returnBool(false);
}
/// __builtin_complex(Float A, float B);
static bool interp__builtin_complex(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg2 = S.Stk.pop<Floating>();
const Floating &Arg1 = S.Stk.pop<Floating>();
Pointer &Result = S.Stk.peek<Pointer>();
Result.elem<Floating>(0) = Arg1;
Result.elem<Floating>(1) = Arg2;
Result.initializeAllElements();
return true;
}
/// __builtin_is_aligned()
/// __builtin_align_up()
/// __builtin_align_down()
/// The first parameter is either an integer or a pointer.
/// The second parameter is the requested alignment as an integer.
static bool interp__builtin_is_aligned_up_down(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call,
unsigned BuiltinOp) {
const APSInt &Alignment = popToAPSInt(S, Call->getArg(1));
if (Alignment < 0 || !Alignment.isPowerOf2()) {
S.FFDiag(Call, diag::note_constexpr_invalid_alignment) << Alignment;
return false;
}
unsigned SrcWidth = S.getASTContext().getIntWidth(Call->getArg(0)->getType());
APSInt MaxValue(APInt::getOneBitSet(SrcWidth, SrcWidth - 1));
if (APSInt::compareValues(Alignment, MaxValue) > 0) {
S.FFDiag(Call, diag::note_constexpr_alignment_too_big)
<< MaxValue << Call->getArg(0)->getType() << Alignment;
return false;
}
// The first parameter is either an integer or a pointer.
PrimType FirstArgT = *S.Ctx.classify(Call->getArg(0));
if (isIntegralType(FirstArgT)) {
const APSInt &Src = popToAPSInt(S.Stk, FirstArgT);
APInt AlignMinusOne = Alignment.extOrTrunc(Src.getBitWidth()) - 1;
if (BuiltinOp == Builtin::BI__builtin_align_up) {
APSInt AlignedVal =
APSInt((Src + AlignMinusOne) & ~AlignMinusOne, Src.isUnsigned());
pushInteger(S, AlignedVal, Call->getType());
} else if (BuiltinOp == Builtin::BI__builtin_align_down) {
APSInt AlignedVal = APSInt(Src & ~AlignMinusOne, Src.isUnsigned());
pushInteger(S, AlignedVal, Call->getType());
} else {
assert(*S.Ctx.classify(Call->getType()) == PT_Bool);
S.Stk.push<Boolean>((Src & AlignMinusOne) == 0);
}
return true;
}
assert(FirstArgT == PT_Ptr);
const Pointer &Ptr = S.Stk.pop<Pointer>();
if (!Ptr.isBlockPointer())
return false;
unsigned PtrOffset = Ptr.getIndex();
CharUnits BaseAlignment =
S.getASTContext().getDeclAlign(Ptr.getDeclDesc()->asValueDecl());
CharUnits PtrAlign =
BaseAlignment.alignmentAtOffset(CharUnits::fromQuantity(PtrOffset));
if (BuiltinOp == Builtin::BI__builtin_is_aligned) {
if (PtrAlign.getQuantity() >= Alignment) {
S.Stk.push<Boolean>(true);
return true;
}
// If the alignment is not known to be sufficient, some cases could still
// be aligned at run time. However, if the requested alignment is less or
// equal to the base alignment and the offset is not aligned, we know that
// the run-time value can never be aligned.
if (BaseAlignment.getQuantity() >= Alignment &&
PtrAlign.getQuantity() < Alignment) {
S.Stk.push<Boolean>(false);
return true;
}
S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_compute)
<< Alignment;
return false;
}
assert(BuiltinOp == Builtin::BI__builtin_align_down ||
BuiltinOp == Builtin::BI__builtin_align_up);
// For align_up/align_down, we can return the same value if the alignment
// is known to be greater or equal to the requested value.
if (PtrAlign.getQuantity() >= Alignment) {
S.Stk.push<Pointer>(Ptr);
return true;
}
// The alignment could be greater than the minimum at run-time, so we cannot
// infer much about the resulting pointer value. One case is possible:
// For `_Alignas(32) char buf[N]; __builtin_align_down(&buf[idx], 32)` we
// can infer the correct index if the requested alignment is smaller than
// the base alignment so we can perform the computation on the offset.
if (BaseAlignment.getQuantity() >= Alignment) {
assert(Alignment.getBitWidth() <= 64 &&
"Cannot handle > 64-bit address-space");
uint64_t Alignment64 = Alignment.getZExtValue();
CharUnits NewOffset =
CharUnits::fromQuantity(BuiltinOp == Builtin::BI__builtin_align_down
? llvm::alignDown(PtrOffset, Alignment64)
: llvm::alignTo(PtrOffset, Alignment64));
S.Stk.push<Pointer>(Ptr.atIndex(NewOffset.getQuantity()));
return true;
}
// Otherwise, we cannot constant-evaluate the result.
S.FFDiag(Call->getArg(0), diag::note_constexpr_alignment_adjust) << Alignment;
return false;
}
/// __builtin_assume_aligned(Ptr, Alignment[, ExtraOffset])
static bool interp__builtin_assume_aligned(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
std::optional<APSInt> ExtraOffset;
if (Call->getNumArgs() == 3)
ExtraOffset = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(2)));
APSInt Alignment = popToAPSInt(S.Stk, *S.Ctx.classify(Call->getArg(1)));
const Pointer &Ptr = S.Stk.pop<Pointer>();
CharUnits Align = CharUnits::fromQuantity(Alignment.getZExtValue());
// If there is a base object, then it must have the correct alignment.
if (Ptr.isBlockPointer()) {
CharUnits BaseAlignment;
if (const auto *VD = Ptr.getDeclDesc()->asValueDecl())
BaseAlignment = S.getASTContext().getDeclAlign(VD);
else if (const auto *E = Ptr.getDeclDesc()->asExpr())
BaseAlignment = GetAlignOfExpr(S.getASTContext(), E, UETT_AlignOf);
if (BaseAlignment < Align) {
S.CCEDiag(Call->getArg(0),
diag::note_constexpr_baa_insufficient_alignment)
<< 0 << BaseAlignment.getQuantity() << Align.getQuantity();
return false;
}
}
APValue AV = Ptr.toAPValue(S.getASTContext());
CharUnits AVOffset = AV.getLValueOffset();
if (ExtraOffset)
AVOffset -= CharUnits::fromQuantity(ExtraOffset->getZExtValue());
if (AVOffset.alignTo(Align) != AVOffset) {
if (Ptr.isBlockPointer())
S.CCEDiag(Call->getArg(0),
diag::note_constexpr_baa_insufficient_alignment)
<< 1 << AVOffset.getQuantity() << Align.getQuantity();
else
S.CCEDiag(Call->getArg(0),
diag::note_constexpr_baa_value_insufficient_alignment)
<< AVOffset.getQuantity() << Align.getQuantity();
return false;
}
S.Stk.push<Pointer>(Ptr);
return true;
}
/// (CarryIn, LHS, RHS, Result)
static bool interp__builtin_ia32_addcarry_subborrow(InterpState &S,
CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call,
unsigned BuiltinOp) {
if (Call->getNumArgs() != 4 || !Call->getArg(0)->getType()->isIntegerType() ||
!Call->getArg(1)->getType()->isIntegerType() ||
!Call->getArg(2)->getType()->isIntegerType())
return false;
const Pointer &CarryOutPtr = S.Stk.pop<Pointer>();
APSInt RHS = popToAPSInt(S, Call->getArg(2));
APSInt LHS = popToAPSInt(S, Call->getArg(1));
APSInt CarryIn = popToAPSInt(S, Call->getArg(0));
bool IsAdd = BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u32 ||
BuiltinOp == clang::X86::BI__builtin_ia32_addcarryx_u64;
unsigned BitWidth = LHS.getBitWidth();
unsigned CarryInBit = CarryIn.ugt(0) ? 1 : 0;
APInt ExResult =
IsAdd ? (LHS.zext(BitWidth + 1) + (RHS.zext(BitWidth + 1) + CarryInBit))
: (LHS.zext(BitWidth + 1) - (RHS.zext(BitWidth + 1) + CarryInBit));
APInt Result = ExResult.extractBits(BitWidth, 0);
APSInt CarryOut =
APSInt(ExResult.extractBits(1, BitWidth), /*IsUnsigned=*/true);
QualType CarryOutType = Call->getArg(3)->getType()->getPointeeType();
PrimType CarryOutT = *S.getContext().classify(CarryOutType);
assignInteger(S, CarryOutPtr, CarryOutT, APSInt(std::move(Result), true));
pushInteger(S, CarryOut, Call->getType());
return true;
}
static bool interp__builtin_os_log_format_buffer_size(InterpState &S,
CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
analyze_os_log::OSLogBufferLayout Layout;
analyze_os_log::computeOSLogBufferLayout(S.getASTContext(), Call, Layout);
pushInteger(S, Layout.size().getQuantity(), Call->getType());
return true;
}
static bool
interp__builtin_ptrauth_string_discriminator(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const auto &Ptr = S.Stk.pop<Pointer>();
assert(Ptr.getFieldDesc()->isPrimitiveArray());
// This should be created for a StringLiteral, so should alway shold at least
// one array element.
assert(Ptr.getFieldDesc()->getNumElems() >= 1);
StringRef R(&Ptr.deref<char>(), Ptr.getFieldDesc()->getNumElems() - 1);
uint64_t Result = getPointerAuthStableSipHash(R);
pushInteger(S, Result, Call->getType());
return true;
}
static bool interp__builtin_infer_alloc_token(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const ASTContext &ASTCtx = S.getASTContext();
uint64_t BitWidth = ASTCtx.getTypeSize(ASTCtx.getSizeType());
auto Mode =
ASTCtx.getLangOpts().AllocTokenMode.value_or(llvm::DefaultAllocTokenMode);
auto MaxTokensOpt = ASTCtx.getLangOpts().AllocTokenMax;
uint64_t MaxTokens =
MaxTokensOpt.value_or(0) ? *MaxTokensOpt : (~0ULL >> (64 - BitWidth));
// We do not read any of the arguments; discard them.
for (int I = Call->getNumArgs() - 1; I >= 0; --I)
discard(S.Stk, *S.getContext().classify(Call->getArg(I)));
// Note: Type inference from a surrounding cast is not supported in
// constexpr evaluation.
QualType AllocType = infer_alloc::inferPossibleType(Call, ASTCtx, nullptr);
if (AllocType.isNull()) {
S.CCEDiag(Call,
diag::note_constexpr_infer_alloc_token_type_inference_failed);
return false;
}
auto ATMD = infer_alloc::getAllocTokenMetadata(AllocType, ASTCtx);
if (!ATMD) {
S.CCEDiag(Call, diag::note_constexpr_infer_alloc_token_no_metadata);
return false;
}
auto MaybeToken = llvm::getAllocToken(Mode, *ATMD, MaxTokens);
if (!MaybeToken) {
S.CCEDiag(Call, diag::note_constexpr_infer_alloc_token_stateful_mode);
return false;
}
pushInteger(S, llvm::APInt(BitWidth, *MaybeToken), ASTCtx.getSizeType());
return true;
}
static bool interp__builtin_operator_new(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
// A call to __operator_new is only valid within std::allocate<>::allocate.
// Walk up the call stack to find the appropriate caller and get the
// element type from it.
auto [NewCall, ElemType] = S.getStdAllocatorCaller("allocate");
if (ElemType.isNull()) {
S.FFDiag(Call, S.getLangOpts().CPlusPlus20
? diag::note_constexpr_new_untyped
: diag::note_constexpr_new);
return false;
}
assert(NewCall);
if (ElemType->isIncompleteType() || ElemType->isFunctionType()) {
S.FFDiag(Call, diag::note_constexpr_new_not_complete_object_type)
<< (ElemType->isIncompleteType() ? 0 : 1) << ElemType;
return false;
}
// We only care about the first parameter (the size), so discard all the
// others.
{
unsigned NumArgs = Call->getNumArgs();
assert(NumArgs >= 1);
// The std::nothrow_t arg never gets put on the stack.
if (Call->getArg(NumArgs - 1)->getType()->isNothrowT())
--NumArgs;
auto Args = ArrayRef(Call->getArgs(), Call->getNumArgs());
// First arg is needed.
Args = Args.drop_front();
// Discard the rest.
for (const Expr *Arg : Args)
discard(S.Stk, *S.getContext().classify(Arg));
}
APSInt Bytes = popToAPSInt(S, Call->getArg(0));
CharUnits ElemSize = S.getASTContext().getTypeSizeInChars(ElemType);
assert(!ElemSize.isZero());
// Divide the number of bytes by sizeof(ElemType), so we get the number of
// elements we should allocate.
APInt NumElems, Remainder;
APInt ElemSizeAP(Bytes.getBitWidth(), ElemSize.getQuantity());
APInt::udivrem(Bytes, ElemSizeAP, NumElems, Remainder);
if (Remainder != 0) {
// This likely indicates a bug in the implementation of 'std::allocator'.
S.FFDiag(Call, diag::note_constexpr_operator_new_bad_size)
<< Bytes << APSInt(ElemSizeAP, true) << ElemType;
return false;
}
// NB: The same check we're using in CheckArraySize()
if (NumElems.getActiveBits() >
ConstantArrayType::getMaxSizeBits(S.getASTContext()) ||
NumElems.ugt(Descriptor::MaxArrayElemBytes / ElemSize.getQuantity())) {
// FIXME: NoThrow check?
const SourceInfo &Loc = S.Current->getSource(OpPC);
S.FFDiag(Loc, diag::note_constexpr_new_too_large)
<< NumElems.getZExtValue();
return false;
}
if (!CheckArraySize(S, OpPC, NumElems.getZExtValue()))
return false;
bool IsArray = NumElems.ugt(1);
OptPrimType ElemT = S.getContext().classify(ElemType);
DynamicAllocator &Allocator = S.getAllocator();
if (ElemT) {
Block *B =
Allocator.allocate(NewCall, *ElemT, NumElems.getZExtValue(),
S.Ctx.getEvalID(), DynamicAllocator::Form::Operator);
assert(B);
S.Stk.push<Pointer>(Pointer(B).atIndex(0));
return true;
}
assert(!ElemT);
// Composite arrays
if (IsArray) {
const Descriptor *Desc =
S.P.createDescriptor(NewCall, ElemType.getTypePtr(), std::nullopt);
Block *B =
Allocator.allocate(Desc, NumElems.getZExtValue(), S.Ctx.getEvalID(),
DynamicAllocator::Form::Operator);
assert(B);
S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());
return true;
}
// Records. Still allocate them as single-element arrays.
QualType AllocType = S.getASTContext().getConstantArrayType(
ElemType, NumElems, nullptr, ArraySizeModifier::Normal, 0);
const Descriptor *Desc = S.P.createDescriptor(NewCall, AllocType.getTypePtr(),
Descriptor::InlineDescMD);
Block *B = Allocator.allocate(Desc, S.getContext().getEvalID(),
DynamicAllocator::Form::Operator);
assert(B);
S.Stk.push<Pointer>(Pointer(B).atIndex(0).narrow());
return true;
}
static bool interp__builtin_operator_delete(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Expr *Source = nullptr;
const Block *BlockToDelete = nullptr;
if (S.checkingPotentialConstantExpression()) {
S.Stk.discard<Pointer>();
return false;
}
// This is permitted only within a call to std::allocator<T>::deallocate.
if (!S.getStdAllocatorCaller("deallocate")) {
S.FFDiag(Call);
S.Stk.discard<Pointer>();
return true;
}
{
const Pointer &Ptr = S.Stk.pop<Pointer>();
if (Ptr.isZero()) {
S.CCEDiag(Call, diag::note_constexpr_deallocate_null);
return true;
}
Source = Ptr.getDeclDesc()->asExpr();
BlockToDelete = Ptr.block();
if (!BlockToDelete->isDynamic()) {
S.FFDiag(Call, diag::note_constexpr_delete_not_heap_alloc)
<< Ptr.toDiagnosticString(S.getASTContext());
if (const auto *D = Ptr.getFieldDesc()->asDecl())
S.Note(D->getLocation(), diag::note_declared_at);
}
}
assert(BlockToDelete);
DynamicAllocator &Allocator = S.getAllocator();
const Descriptor *BlockDesc = BlockToDelete->getDescriptor();
std::optional<DynamicAllocator::Form> AllocForm =
Allocator.getAllocationForm(Source);
if (!Allocator.deallocate(Source, BlockToDelete, S)) {
// Nothing has been deallocated, this must be a double-delete.
const SourceInfo &Loc = S.Current->getSource(OpPC);
S.FFDiag(Loc, diag::note_constexpr_double_delete);
return false;
}
assert(AllocForm);
return CheckNewDeleteForms(
S, OpPC, *AllocForm, DynamicAllocator::Form::Operator, BlockDesc, Source);
}
static bool interp__builtin_arithmetic_fence(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const Floating &Arg0 = S.Stk.pop<Floating>();
S.Stk.push<Floating>(Arg0);
return true;
}
static bool interp__builtin_vector_reduce(InterpState &S, CodePtr OpPC,
const CallExpr *Call, unsigned ID) {
const Pointer &Arg = S.Stk.pop<Pointer>();
assert(Arg.getFieldDesc()->isPrimitiveArray());
QualType ElemType = Arg.getFieldDesc()->getElemQualType();
assert(Call->getType() == ElemType);
PrimType ElemT = *S.getContext().classify(ElemType);
unsigned NumElems = Arg.getNumElems();
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
T Result = Arg.elem<T>(0);
unsigned BitWidth = Result.bitWidth();
for (unsigned I = 1; I != NumElems; ++I) {
T Elem = Arg.elem<T>(I);
T PrevResult = Result;
if (ID == Builtin::BI__builtin_reduce_add) {
if (T::add(Result, Elem, BitWidth, &Result)) {
unsigned OverflowBits = BitWidth + 1;
(void)handleOverflow(S, OpPC,
(PrevResult.toAPSInt(OverflowBits) +
Elem.toAPSInt(OverflowBits)));
return false;
}
} else if (ID == Builtin::BI__builtin_reduce_mul) {
if (T::mul(Result, Elem, BitWidth, &Result)) {
unsigned OverflowBits = BitWidth * 2;
(void)handleOverflow(S, OpPC,
(PrevResult.toAPSInt(OverflowBits) *
Elem.toAPSInt(OverflowBits)));
return false;
}
} else if (ID == Builtin::BI__builtin_reduce_and) {
(void)T::bitAnd(Result, Elem, BitWidth, &Result);
} else if (ID == Builtin::BI__builtin_reduce_or) {
(void)T::bitOr(Result, Elem, BitWidth, &Result);
} else if (ID == Builtin::BI__builtin_reduce_xor) {
(void)T::bitXor(Result, Elem, BitWidth, &Result);
} else if (ID == Builtin::BI__builtin_reduce_min) {
if (Elem < Result)
Result = Elem;
} else if (ID == Builtin::BI__builtin_reduce_max) {
if (Elem > Result)
Result = Elem;
} else {
llvm_unreachable("Unhandled vector reduce builtin");
}
}
pushInteger(S, Result.toAPSInt(), Call->getType());
});
return true;
}
static bool interp__builtin_elementwise_abs(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call,
unsigned BuiltinID) {
assert(Call->getNumArgs() == 1);
QualType Ty = Call->getArg(0)->getType();
if (Ty->isIntegerType()) {
APSInt Val = popToAPSInt(S, Call->getArg(0));
pushInteger(S, Val.abs(), Call->getType());
return true;
}
if (Ty->isFloatingType()) {
Floating Val = S.Stk.pop<Floating>();
Floating Result = abs(S, Val);
S.Stk.push<Floating>(Result);
return true;
}
// Otherwise, the argument must be a vector.
assert(Call->getArg(0)->getType()->isVectorType());
const Pointer &Arg = S.Stk.pop<Pointer>();
assert(Arg.getFieldDesc()->isPrimitiveArray());
const Pointer &Dst = S.Stk.peek<Pointer>();
assert(Dst.getFieldDesc()->isPrimitiveArray());
assert(Arg.getFieldDesc()->getNumElems() ==
Dst.getFieldDesc()->getNumElems());
QualType ElemType = Arg.getFieldDesc()->getElemQualType();
PrimType ElemT = *S.getContext().classify(ElemType);
unsigned NumElems = Arg.getNumElems();
// we can either have a vector of integer or a vector of floating point
for (unsigned I = 0; I != NumElems; ++I) {
if (ElemType->isIntegerType()) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
Dst.elem<T>(I) = T::from(static_cast<T>(
APSInt(Arg.elem<T>(I).toAPSInt().abs(),
ElemType->isUnsignedIntegerOrEnumerationType())));
});
} else {
Floating Val = Arg.elem<Floating>(I);
Dst.elem<Floating>(I) = abs(S, Val);
}
}
Dst.initializeAllElements();
return true;
}
/// Can be called with an integer or vector as the first and only parameter.
static bool interp__builtin_elementwise_countzeroes(InterpState &S,
CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call,
unsigned BuiltinID) {
bool HasZeroArg = Call->getNumArgs() == 2;
bool IsCTTZ = BuiltinID == Builtin::BI__builtin_elementwise_ctzg;
assert(Call->getNumArgs() == 1 || HasZeroArg);
if (Call->getArg(0)->getType()->isIntegerType()) {
PrimType ArgT = *S.getContext().classify(Call->getArg(0)->getType());
APSInt Val = popToAPSInt(S.Stk, ArgT);
std::optional<APSInt> ZeroVal;
if (HasZeroArg) {
ZeroVal = Val;
Val = popToAPSInt(S.Stk, ArgT);
}
if (Val.isZero()) {
if (ZeroVal) {
pushInteger(S, *ZeroVal, Call->getType());
return true;
}
// If we haven't been provided the second argument, the result is
// undefined
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_countzeroes_zero)
<< /*IsTrailing=*/IsCTTZ;
return false;
}
if (BuiltinID == Builtin::BI__builtin_elementwise_clzg) {
pushInteger(S, Val.countLeadingZeros(), Call->getType());
} else {
pushInteger(S, Val.countTrailingZeros(), Call->getType());
}
return true;
}
// Otherwise, the argument must be a vector.
const ASTContext &ASTCtx = S.getASTContext();
Pointer ZeroArg;
if (HasZeroArg) {
assert(Call->getArg(1)->getType()->isVectorType() &&
ASTCtx.hasSameUnqualifiedType(Call->getArg(0)->getType(),
Call->getArg(1)->getType()));
(void)ASTCtx;
ZeroArg = S.Stk.pop<Pointer>();
assert(ZeroArg.getFieldDesc()->isPrimitiveArray());
}
assert(Call->getArg(0)->getType()->isVectorType());
const Pointer &Arg = S.Stk.pop<Pointer>();
assert(Arg.getFieldDesc()->isPrimitiveArray());
const Pointer &Dst = S.Stk.peek<Pointer>();
assert(Dst.getFieldDesc()->isPrimitiveArray());
assert(Arg.getFieldDesc()->getNumElems() ==
Dst.getFieldDesc()->getNumElems());
QualType ElemType = Arg.getFieldDesc()->getElemQualType();
PrimType ElemT = *S.getContext().classify(ElemType);
unsigned NumElems = Arg.getNumElems();
// FIXME: Reading from uninitialized vector elements?
for (unsigned I = 0; I != NumElems; ++I) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
APInt EltVal = Arg.atIndex(I).deref<T>().toAPSInt();
if (EltVal.isZero()) {
if (HasZeroArg) {
Dst.atIndex(I).deref<T>() = ZeroArg.atIndex(I).deref<T>();
} else {
// If we haven't been provided the second argument, the result is
// undefined
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_countzeroes_zero)
<< /*IsTrailing=*/IsCTTZ;
return false;
}
} else if (IsCTTZ) {
Dst.atIndex(I).deref<T>() = T::from(EltVal.countTrailingZeros());
} else {
Dst.atIndex(I).deref<T>() = T::from(EltVal.countLeadingZeros());
}
Dst.atIndex(I).initialize();
});
}
return true;
}
static bool interp__builtin_memcpy(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call, unsigned ID) {
assert(Call->getNumArgs() == 3);
const ASTContext &ASTCtx = S.getASTContext();
APSInt Size = popToAPSInt(S, Call->getArg(2));
Pointer SrcPtr = S.Stk.pop<Pointer>().expand();
Pointer DestPtr = S.Stk.pop<Pointer>().expand();
assert(!Size.isSigned() && "memcpy and friends take an unsigned size");
if (ID == Builtin::BImemcpy || ID == Builtin::BImemmove)
diagnoseNonConstexprBuiltin(S, OpPC, ID);
bool Move =
(ID == Builtin::BI__builtin_memmove || ID == Builtin::BImemmove ||
ID == Builtin::BI__builtin_wmemmove || ID == Builtin::BIwmemmove);
bool WChar = ID == Builtin::BIwmemcpy || ID == Builtin::BIwmemmove ||
ID == Builtin::BI__builtin_wmemcpy ||
ID == Builtin::BI__builtin_wmemmove;
// If the size is zero, we treat this as always being a valid no-op.
if (Size.isZero()) {
S.Stk.push<Pointer>(DestPtr);
return true;
}
if (SrcPtr.isZero() || DestPtr.isZero()) {
Pointer DiagPtr = (SrcPtr.isZero() ? SrcPtr : DestPtr);
S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
<< /*IsMove=*/Move << /*IsWchar=*/WChar << !SrcPtr.isZero()
<< DiagPtr.toDiagnosticString(ASTCtx);
return false;
}
// Diagnose integral src/dest pointers specially.
if (SrcPtr.isIntegralPointer() || DestPtr.isIntegralPointer()) {
std::string DiagVal = "(void *)";
DiagVal += SrcPtr.isIntegralPointer()
? std::to_string(SrcPtr.getIntegerRepresentation())
: std::to_string(DestPtr.getIntegerRepresentation());
S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_null)
<< Move << WChar << DestPtr.isIntegralPointer() << DiagVal;
return false;
}
// Can't read from dummy pointers.
if (DestPtr.isDummy() || SrcPtr.isDummy())
return false;
if (DestPtr.getType()->isIncompleteType()) {
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_memcpy_incomplete_type)
<< Move << DestPtr.getType();
return false;
}
if (SrcPtr.getType()->isIncompleteType()) {
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_memcpy_incomplete_type)
<< Move << SrcPtr.getType();
return false;
}
QualType DestElemType = getElemType(DestPtr);
if (DestElemType->isIncompleteType()) {
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_memcpy_incomplete_type)
<< Move << DestElemType;
return false;
}
size_t RemainingDestElems;
if (DestPtr.getFieldDesc()->isArray()) {
RemainingDestElems = DestPtr.isUnknownSizeArray()
? 0
: (DestPtr.getNumElems() - DestPtr.getIndex());
} else {
RemainingDestElems = 1;
}
unsigned DestElemSize = ASTCtx.getTypeSizeInChars(DestElemType).getQuantity();
if (WChar) {
uint64_t WCharSize =
ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();
Size *= APSInt(APInt(Size.getBitWidth(), WCharSize, /*IsSigned=*/false),
/*IsUnsigend=*/true);
}
if (Size.urem(DestElemSize) != 0) {
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_memcpy_unsupported)
<< Move << WChar << 0 << DestElemType << Size << DestElemSize;
return false;
}
QualType SrcElemType = getElemType(SrcPtr);
size_t RemainingSrcElems;
if (SrcPtr.getFieldDesc()->isArray()) {
RemainingSrcElems = SrcPtr.isUnknownSizeArray()
? 0
: (SrcPtr.getNumElems() - SrcPtr.getIndex());
} else {
RemainingSrcElems = 1;
}
unsigned SrcElemSize = ASTCtx.getTypeSizeInChars(SrcElemType).getQuantity();
if (!ASTCtx.hasSameUnqualifiedType(DestElemType, SrcElemType)) {
S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_type_pun)
<< Move << SrcElemType << DestElemType;
return false;
}
if (!DestElemType.isTriviallyCopyableType(ASTCtx)) {
S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_nontrivial)
<< Move << DestElemType;
return false;
}
// Check if we have enough elements to read from and write to.
size_t RemainingDestBytes = RemainingDestElems * DestElemSize;
size_t RemainingSrcBytes = RemainingSrcElems * SrcElemSize;
if (Size.ugt(RemainingDestBytes) || Size.ugt(RemainingSrcBytes)) {
APInt N = Size.udiv(DestElemSize);
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_memcpy_unsupported)
<< Move << WChar << (Size.ugt(RemainingSrcBytes) ? 1 : 2)
<< DestElemType << toString(N, 10, /*Signed=*/false);
return false;
}
// Check for overlapping memory regions.
if (!Move && Pointer::pointToSameBlock(SrcPtr, DestPtr)) {
// Remove base casts.
Pointer SrcP = SrcPtr;
while (SrcP.isBaseClass())
SrcP = SrcP.getBase();
Pointer DestP = DestPtr;
while (DestP.isBaseClass())
DestP = DestP.getBase();
unsigned SrcIndex = SrcP.expand().getIndex() * SrcP.elemSize();
unsigned DstIndex = DestP.expand().getIndex() * DestP.elemSize();
unsigned N = Size.getZExtValue();
if ((SrcIndex <= DstIndex && (SrcIndex + N) > DstIndex) ||
(DstIndex <= SrcIndex && (DstIndex + N) > SrcIndex)) {
S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_memcpy_overlap)
<< /*IsWChar=*/false;
return false;
}
}
assert(Size.getZExtValue() % DestElemSize == 0);
if (!DoMemcpy(S, OpPC, SrcPtr, DestPtr, Bytes(Size.getZExtValue()).toBits()))
return false;
S.Stk.push<Pointer>(DestPtr);
return true;
}
/// Determine if T is a character type for which we guarantee that
/// sizeof(T) == 1.
static bool isOneByteCharacterType(QualType T) {
return T->isCharType() || T->isChar8Type();
}
static bool interp__builtin_memcmp(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call, unsigned ID) {
assert(Call->getNumArgs() == 3);
const APSInt &Size = popToAPSInt(S, Call->getArg(2));
const Pointer &PtrB = S.Stk.pop<Pointer>();
const Pointer &PtrA = S.Stk.pop<Pointer>();
if (ID == Builtin::BImemcmp || ID == Builtin::BIbcmp ||
ID == Builtin::BIwmemcmp)
diagnoseNonConstexprBuiltin(S, OpPC, ID);
if (Size.isZero()) {
pushInteger(S, 0, Call->getType());
return true;
}
if (!PtrA.isBlockPointer() || !PtrB.isBlockPointer())
return false;
bool IsWide =
(ID == Builtin::BIwmemcmp || ID == Builtin::BI__builtin_wmemcmp);
const ASTContext &ASTCtx = S.getASTContext();
QualType ElemTypeA = getElemType(PtrA);
QualType ElemTypeB = getElemType(PtrB);
// FIXME: This is an arbitrary limitation the current constant interpreter
// had. We could remove this.
if (!IsWide && (!isOneByteCharacterType(ElemTypeA) ||
!isOneByteCharacterType(ElemTypeB))) {
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_memcmp_unsupported)
<< ASTCtx.BuiltinInfo.getQuotedName(ID) << PtrA.getType()
<< PtrB.getType();
return false;
}
if (PtrA.isDummy() || PtrB.isDummy())
return false;
// Now, read both pointers to a buffer and compare those.
BitcastBuffer BufferA(
Bits(ASTCtx.getTypeSize(ElemTypeA) * PtrA.getNumElems()));
readPointerToBuffer(S.getContext(), PtrA, BufferA, false);
// FIXME: The swapping here is UNDOING something we do when reading the
// data into the buffer.
if (ASTCtx.getTargetInfo().isBigEndian())
swapBytes(BufferA.Data.get(), BufferA.byteSize().getQuantity());
BitcastBuffer BufferB(
Bits(ASTCtx.getTypeSize(ElemTypeB) * PtrB.getNumElems()));
readPointerToBuffer(S.getContext(), PtrB, BufferB, false);
// FIXME: The swapping here is UNDOING something we do when reading the
// data into the buffer.
if (ASTCtx.getTargetInfo().isBigEndian())
swapBytes(BufferB.Data.get(), BufferB.byteSize().getQuantity());
size_t MinBufferSize = std::min(BufferA.byteSize().getQuantity(),
BufferB.byteSize().getQuantity());
unsigned ElemSize = 1;
if (IsWide)
ElemSize = ASTCtx.getTypeSizeInChars(ASTCtx.getWCharType()).getQuantity();
// The Size given for the wide variants is in wide-char units. Convert it
// to bytes.
size_t ByteSize = Size.getZExtValue() * ElemSize;
size_t CmpSize = std::min(MinBufferSize, ByteSize);
for (size_t I = 0; I != CmpSize; I += ElemSize) {
if (IsWide) {
INT_TYPE_SWITCH(*S.getContext().classify(ASTCtx.getWCharType()), {
T A = *reinterpret_cast<T *>(BufferA.atByte(I));
T B = *reinterpret_cast<T *>(BufferB.atByte(I));
if (A < B) {
pushInteger(S, -1, Call->getType());
return true;
}
if (A > B) {
pushInteger(S, 1, Call->getType());
return true;
}
});
} else {
std::byte A = BufferA.deref<std::byte>(Bytes(I));
std::byte B = BufferB.deref<std::byte>(Bytes(I));
if (A < B) {
pushInteger(S, -1, Call->getType());
return true;
}
if (A > B) {
pushInteger(S, 1, Call->getType());
return true;
}
}
}
// We compared CmpSize bytes above. If the limiting factor was the Size
// passed, we're done and the result is equality (0).
if (ByteSize <= CmpSize) {
pushInteger(S, 0, Call->getType());
return true;
}
// However, if we read all the available bytes but were instructed to read
// even more, diagnose this as a "read of dereferenced one-past-the-end
// pointer". This is what would happen if we called CheckLoad() on every array
// element.
S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_past_end)
<< AK_Read << S.Current->getRange(OpPC);
return false;
}
// __builtin_memchr(ptr, int, int)
// __builtin_strchr(ptr, int)
static bool interp__builtin_memchr(InterpState &S, CodePtr OpPC,
const CallExpr *Call, unsigned ID) {
if (ID == Builtin::BImemchr || ID == Builtin::BIwcschr ||
ID == Builtin::BIstrchr || ID == Builtin::BIwmemchr)
diagnoseNonConstexprBuiltin(S, OpPC, ID);
std::optional<APSInt> MaxLength;
if (Call->getNumArgs() == 3)
MaxLength = popToAPSInt(S, Call->getArg(2));
APSInt Desired = popToAPSInt(S, Call->getArg(1));
const Pointer &Ptr = S.Stk.pop<Pointer>();
if (MaxLength && MaxLength->isZero()) {
S.Stk.push<Pointer>();
return true;
}
if (Ptr.isDummy()) {
if (Ptr.getType()->isIncompleteType())
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_ltor_incomplete_type)
<< Ptr.getType();
return false;
}
// Null is only okay if the given size is 0.
if (Ptr.isZero()) {
S.FFDiag(S.Current->getSource(OpPC), diag::note_constexpr_access_null)
<< AK_Read;
return false;
}
QualType ElemTy = Ptr.getFieldDesc()->isArray()
? Ptr.getFieldDesc()->getElemQualType()
: Ptr.getFieldDesc()->getType();
bool IsRawByte = ID == Builtin::BImemchr || ID == Builtin::BI__builtin_memchr;
// Give up on byte-oriented matching against multibyte elements.
if (IsRawByte && !isOneByteCharacterType(ElemTy)) {
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_memchr_unsupported)
<< S.getASTContext().BuiltinInfo.getQuotedName(ID) << ElemTy;
return false;
}
if (ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr) {
int64_t DesiredTrunc;
if (S.getASTContext().CharTy->isSignedIntegerType())
DesiredTrunc =
Desired.trunc(S.getASTContext().getCharWidth()).getSExtValue();
else
DesiredTrunc =
Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();
// strchr compares directly to the passed integer, and therefore
// always fails if given an int that is not a char.
if (Desired != DesiredTrunc) {
S.Stk.push<Pointer>();
return true;
}
}
uint64_t DesiredVal;
if (ID == Builtin::BIwmemchr || ID == Builtin::BI__builtin_wmemchr ||
ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr) {
// wcschr and wmemchr are given a wchar_t to look for. Just use it.
DesiredVal = Desired.getZExtValue();
} else {
DesiredVal = Desired.trunc(S.getASTContext().getCharWidth()).getZExtValue();
}
bool StopAtZero =
(ID == Builtin::BIstrchr || ID == Builtin::BI__builtin_strchr ||
ID == Builtin::BIwcschr || ID == Builtin::BI__builtin_wcschr);
PrimType ElemT =
IsRawByte ? PT_Sint8 : *S.getContext().classify(getElemType(Ptr));
size_t Index = Ptr.getIndex();
size_t Step = 0;
for (;;) {
const Pointer &ElemPtr =
(Index + Step) > 0 ? Ptr.atIndex(Index + Step) : Ptr;
if (!CheckLoad(S, OpPC, ElemPtr))
return false;
uint64_t V;
INT_TYPE_SWITCH_NO_BOOL(
ElemT, { V = static_cast<uint64_t>(ElemPtr.deref<T>().toUnsigned()); });
if (V == DesiredVal) {
S.Stk.push<Pointer>(ElemPtr);
return true;
}
if (StopAtZero && V == 0)
break;
++Step;
if (MaxLength && Step == MaxLength->getZExtValue())
break;
}
S.Stk.push<Pointer>();
return true;
}
static std::optional<unsigned> computeFullDescSize(const ASTContext &ASTCtx,
const Descriptor *Desc) {
if (Desc->isPrimitive())
return ASTCtx.getTypeSizeInChars(Desc->getType()).getQuantity();
if (Desc->isArray())
return ASTCtx.getTypeSizeInChars(Desc->getElemQualType()).getQuantity() *
Desc->getNumElems();
if (Desc->isRecord()) {
// Can't use Descriptor::getType() as that may return a pointer type. Look
// at the decl directly.
return ASTCtx
.getTypeSizeInChars(
ASTCtx.getCanonicalTagType(Desc->ElemRecord->getDecl()))
.getQuantity();
}
return std::nullopt;
}
/// Compute the byte offset of \p Ptr in the full declaration.
static unsigned computePointerOffset(const ASTContext &ASTCtx,
const Pointer &Ptr) {
unsigned Result = 0;
Pointer P = Ptr;
while (P.isField() || P.isArrayElement()) {
P = P.expand();
const Descriptor *D = P.getFieldDesc();
if (P.isArrayElement()) {
unsigned ElemSize =
ASTCtx.getTypeSizeInChars(D->getElemQualType()).getQuantity();
if (P.isOnePastEnd())
Result += ElemSize * P.getNumElems();
else
Result += ElemSize * P.getIndex();
P = P.expand().getArray();
} else if (P.isBaseClass()) {
const auto *RD = cast<CXXRecordDecl>(D->asDecl());
bool IsVirtual = Ptr.isVirtualBaseClass();
P = P.getBase();
const Record *BaseRecord = P.getRecord();
const ASTRecordLayout &Layout =
ASTCtx.getASTRecordLayout(cast<CXXRecordDecl>(BaseRecord->getDecl()));
if (IsVirtual)
Result += Layout.getVBaseClassOffset(RD).getQuantity();
else
Result += Layout.getBaseClassOffset(RD).getQuantity();
} else if (P.isField()) {
const FieldDecl *FD = P.getField();
const ASTRecordLayout &Layout =
ASTCtx.getASTRecordLayout(FD->getParent());
unsigned FieldIndex = FD->getFieldIndex();
uint64_t FieldOffset =
ASTCtx.toCharUnitsFromBits(Layout.getFieldOffset(FieldIndex))
.getQuantity();
Result += FieldOffset;
P = P.getBase();
} else
llvm_unreachable("Unhandled descriptor type");
}
return Result;
}
/// Does Ptr point to the last subobject?
static bool pointsToLastObject(const Pointer &Ptr) {
Pointer P = Ptr;
while (!P.isRoot()) {
if (P.isArrayElement()) {
P = P.expand().getArray();
continue;
}
if (P.isBaseClass()) {
if (P.getRecord()->getNumFields() > 0)
return false;
P = P.getBase();
continue;
}
Pointer Base = P.getBase();
if (const Record *R = Base.getRecord()) {
assert(P.getField());
if (P.getField()->getFieldIndex() != R->getNumFields() - 1)
return false;
}
P = Base;
}
return true;
}
/// Does Ptr point to the last object AND to a flexible array member?
static bool isUserWritingOffTheEnd(const ASTContext &Ctx, const Pointer &Ptr) {
auto isFlexibleArrayMember = [&](const Descriptor *FieldDesc) {
using FAMKind = LangOptions::StrictFlexArraysLevelKind;
FAMKind StrictFlexArraysLevel =
Ctx.getLangOpts().getStrictFlexArraysLevel();
if (StrictFlexArraysLevel == FAMKind::Default)
return true;
unsigned NumElems = FieldDesc->getNumElems();
if (NumElems == 0 && StrictFlexArraysLevel != FAMKind::IncompleteOnly)
return true;
if (NumElems == 1 && StrictFlexArraysLevel == FAMKind::OneZeroOrIncomplete)
return true;
return false;
};
const Descriptor *FieldDesc = Ptr.getFieldDesc();
if (!FieldDesc->isArray())
return false;
return Ptr.isDummy() && pointsToLastObject(Ptr) &&
isFlexibleArrayMember(FieldDesc);
}
static bool interp__builtin_object_size(InterpState &S, CodePtr OpPC,
const InterpFrame *Frame,
const CallExpr *Call) {
const ASTContext &ASTCtx = S.getASTContext();
// From the GCC docs:
// Kind is an integer constant from 0 to 3. If the least significant bit is
// clear, objects are whole variables. If it is set, a closest surrounding
// subobject is considered the object a pointer points to. The second bit
// determines if maximum or minimum of remaining bytes is computed.
unsigned Kind = popToAPSInt(S, Call->getArg(1)).getZExtValue();
assert(Kind <= 3 && "unexpected kind");
bool UseFieldDesc = (Kind & 1u);
bool ReportMinimum = (Kind & 2u);
const Pointer &Ptr = S.Stk.pop<Pointer>();
if (Call->getArg(0)->HasSideEffects(ASTCtx)) {
// "If there are any side effects in them, it returns (size_t) -1
// for type 0 or 1 and (size_t) 0 for type 2 or 3."
pushInteger(S, Kind <= 1 ? -1 : 0, Call->getType());
return true;
}
if (Ptr.isZero() || !Ptr.isBlockPointer())
return false;
// We can't load through pointers.
if (Ptr.isDummy() && Ptr.getType()->isPointerType())
return false;
bool DetermineForCompleteObject = Ptr.getFieldDesc() == Ptr.getDeclDesc();
const Descriptor *DeclDesc = Ptr.getDeclDesc();
assert(DeclDesc);
if (!UseFieldDesc || DetermineForCompleteObject) {
// Lower bound, so we can't fall back to this.
if (ReportMinimum && !DetermineForCompleteObject)
return false;
// Can't read beyond the pointer decl desc.
if (!UseFieldDesc && !ReportMinimum && DeclDesc->getType()->isPointerType())
return false;
} else {
if (isUserWritingOffTheEnd(ASTCtx, Ptr.expand())) {
// If we cannot determine the size of the initial allocation, then we
// can't given an accurate upper-bound. However, we are still able to give
// conservative lower-bounds for Type=3.
if (Kind == 1)
return false;
}
}
const Descriptor *Desc = UseFieldDesc ? Ptr.getFieldDesc() : DeclDesc;
assert(Desc);
std::optional<unsigned> FullSize = computeFullDescSize(ASTCtx, Desc);
if (!FullSize)
return false;
unsigned ByteOffset;
if (UseFieldDesc) {
if (Ptr.isBaseClass())
ByteOffset = computePointerOffset(ASTCtx, Ptr.getBase()) -
computePointerOffset(ASTCtx, Ptr);
else {
if (Ptr.inArray())
ByteOffset =
computePointerOffset(ASTCtx, Ptr) -
computePointerOffset(ASTCtx, Ptr.expand().atIndex(0).narrow());
else
ByteOffset = 0;
}
} else
ByteOffset = computePointerOffset(ASTCtx, Ptr);
assert(ByteOffset <= *FullSize);
unsigned Result = *FullSize - ByteOffset;
pushInteger(S, Result, Call->getType());
return true;
}
static bool interp__builtin_is_within_lifetime(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
if (!S.inConstantContext())
return false;
const Pointer &Ptr = S.Stk.pop<Pointer>();
auto Error = [&](int Diag) {
bool CalledFromStd = false;
const auto *Callee = S.Current->getCallee();
if (Callee && Callee->isInStdNamespace()) {
const IdentifierInfo *Identifier = Callee->getIdentifier();
CalledFromStd = Identifier && Identifier->isStr("is_within_lifetime");
}
S.CCEDiag(CalledFromStd
? S.Current->Caller->getSource(S.Current->getRetPC())
: S.Current->getSource(OpPC),
diag::err_invalid_is_within_lifetime)
<< (CalledFromStd ? "std::is_within_lifetime"
: "__builtin_is_within_lifetime")
<< Diag;
return false;
};
if (Ptr.isZero())
return Error(0);
if (Ptr.isOnePastEnd())
return Error(1);
bool Result = Ptr.getLifetime() != Lifetime::Ended;
if (!Ptr.isActive()) {
Result = false;
} else {
if (!CheckLive(S, OpPC, Ptr, AK_Read))
return false;
if (!CheckMutable(S, OpPC, Ptr))
return false;
if (!CheckDummy(S, OpPC, Ptr.block(), AK_Read))
return false;
}
// Check if we're currently running an initializer.
if (llvm::is_contained(S.InitializingBlocks, Ptr.block()))
return Error(2);
if (S.EvaluatingDecl && Ptr.getDeclDesc()->asVarDecl() == S.EvaluatingDecl)
return Error(2);
pushInteger(S, Result, Call->getType());
return true;
}
static bool interp__builtin_elementwise_int_unaryop(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<APInt(const APSInt &)> Fn) {
assert(Call->getNumArgs() == 1);
// Single integer case.
if (!Call->getArg(0)->getType()->isVectorType()) {
assert(Call->getType()->isIntegerType());
APSInt Src = popToAPSInt(S, Call->getArg(0));
APInt Result = Fn(Src);
pushInteger(S, APSInt(std::move(Result), !Src.isSigned()), Call->getType());
return true;
}
// Vector case.
const Pointer &Arg = S.Stk.pop<Pointer>();
assert(Arg.getFieldDesc()->isPrimitiveArray());
const Pointer &Dst = S.Stk.peek<Pointer>();
assert(Dst.getFieldDesc()->isPrimitiveArray());
assert(Arg.getFieldDesc()->getNumElems() ==
Dst.getFieldDesc()->getNumElems());
QualType ElemType = Arg.getFieldDesc()->getElemQualType();
PrimType ElemT = *S.getContext().classify(ElemType);
unsigned NumElems = Arg.getNumElems();
bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
for (unsigned I = 0; I != NumElems; ++I) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
APSInt Src = Arg.elem<T>(I).toAPSInt();
APInt Result = Fn(Src);
Dst.elem<T>(I) = static_cast<T>(APSInt(std::move(Result), DestUnsigned));
});
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_elementwise_int_binop(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
assert(Call->getNumArgs() == 2);
// Single integer case.
if (!Call->getArg(0)->getType()->isVectorType()) {
assert(!Call->getArg(1)->getType()->isVectorType());
APSInt RHS = popToAPSInt(S, Call->getArg(1));
APSInt LHS = popToAPSInt(S, Call->getArg(0));
APInt Result = Fn(LHS, RHS);
pushInteger(S, APSInt(std::move(Result), !LHS.isSigned()), Call->getType());
return true;
}
const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
assert(VT->getElementType()->isIntegralOrEnumerationType());
PrimType ElemT = *S.getContext().classify(VT->getElementType());
unsigned NumElems = VT->getNumElements();
bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
// Vector + Scalar case.
if (!Call->getArg(1)->getType()->isVectorType()) {
assert(Call->getArg(1)->getType()->isIntegralOrEnumerationType());
APSInt RHS = popToAPSInt(S, Call->getArg(1));
const Pointer &LHS = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
for (unsigned I = 0; I != NumElems; ++I) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
Dst.elem<T>(I) = static_cast<T>(
APSInt(Fn(LHS.elem<T>(I).toAPSInt(), RHS), DestUnsigned));
});
}
Dst.initializeAllElements();
return true;
}
// Vector case.
assert(Call->getArg(0)->getType()->isVectorType() &&
Call->getArg(1)->getType()->isVectorType());
assert(VT->getElementType() ==
Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
assert(VT->getNumElements() ==
Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
assert(VT->getElementType()->isIntegralOrEnumerationType());
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
for (unsigned I = 0; I != NumElems; ++I) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
APSInt Elem1 = LHS.elem<T>(I).toAPSInt();
APSInt Elem2 = RHS.elem<T>(I).toAPSInt();
Dst.elem<T>(I) = static_cast<T>(APSInt(Fn(Elem1, Elem2), DestUnsigned));
});
}
Dst.initializeAllElements();
return true;
}
static bool
interp__builtin_x86_pack(InterpState &S, CodePtr, const CallExpr *E,
llvm::function_ref<APInt(const APSInt &)> PackFn) {
const auto *VT0 = E->getArg(0)->getType()->castAs<VectorType>();
[[maybe_unused]] const auto *VT1 =
E->getArg(1)->getType()->castAs<VectorType>();
assert(VT0 && VT1 && "pack builtin VT0 and VT1 must be VectorType");
assert(VT0->getElementType() == VT1->getElementType() &&
VT0->getNumElements() == VT1->getNumElements() &&
"pack builtin VT0 and VT1 ElementType must be same");
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
const ASTContext &ASTCtx = S.getASTContext();
unsigned SrcBits = ASTCtx.getIntWidth(VT0->getElementType());
unsigned LHSVecLen = VT0->getNumElements();
unsigned SrcPerLane = 128 / SrcBits;
unsigned Lanes = LHSVecLen * SrcBits / 128;
PrimType SrcT = *S.getContext().classify(VT0->getElementType());
PrimType DstT = *S.getContext().classify(getElemType(Dst));
bool IsUnsigend = getElemType(Dst)->isUnsignedIntegerType();
for (unsigned Lane = 0; Lane != Lanes; ++Lane) {
unsigned BaseSrc = Lane * SrcPerLane;
unsigned BaseDst = Lane * (2 * SrcPerLane);
for (unsigned I = 0; I != SrcPerLane; ++I) {
INT_TYPE_SWITCH_NO_BOOL(SrcT, {
APSInt A = LHS.elem<T>(BaseSrc + I).toAPSInt();
APSInt B = RHS.elem<T>(BaseSrc + I).toAPSInt();
assignInteger(S, Dst.atIndex(BaseDst + I), DstT,
APSInt(PackFn(A), IsUnsigend));
assignInteger(S, Dst.atIndex(BaseDst + SrcPerLane + I), DstT,
APSInt(PackFn(B), IsUnsigend));
});
}
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_elementwise_maxmin(InterpState &S, CodePtr OpPC,
const CallExpr *Call,
unsigned BuiltinID) {
assert(Call->getNumArgs() == 2);
QualType Arg0Type = Call->getArg(0)->getType();
// TODO: Support floating-point types.
if (!(Arg0Type->isIntegerType() ||
(Arg0Type->isVectorType() &&
Arg0Type->castAs<VectorType>()->getElementType()->isIntegerType())))
return false;
if (!Arg0Type->isVectorType()) {
assert(!Call->getArg(1)->getType()->isVectorType());
APSInt RHS = popToAPSInt(S, Call->getArg(1));
APSInt LHS = popToAPSInt(S, Arg0Type);
APInt Result;
if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
Result = std::max(LHS, RHS);
} else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
Result = std::min(LHS, RHS);
} else {
llvm_unreachable("Wrong builtin ID");
}
pushInteger(S, APSInt(Result, !LHS.isSigned()), Call->getType());
return true;
}
// Vector case.
assert(Call->getArg(0)->getType()->isVectorType() &&
Call->getArg(1)->getType()->isVectorType());
const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
assert(VT->getElementType() ==
Call->getArg(1)->getType()->castAs<VectorType>()->getElementType());
assert(VT->getNumElements() ==
Call->getArg(1)->getType()->castAs<VectorType>()->getNumElements());
assert(VT->getElementType()->isIntegralOrEnumerationType());
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
PrimType ElemT = *S.getContext().classify(VT->getElementType());
unsigned NumElems = VT->getNumElements();
for (unsigned I = 0; I != NumElems; ++I) {
APSInt Elem1;
APSInt Elem2;
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
Elem1 = LHS.elem<T>(I).toAPSInt();
Elem2 = RHS.elem<T>(I).toAPSInt();
});
APSInt Result;
if (BuiltinID == Builtin::BI__builtin_elementwise_max) {
Result = APSInt(std::max(Elem1, Elem2),
Call->getType()->isUnsignedIntegerOrEnumerationType());
} else if (BuiltinID == Builtin::BI__builtin_elementwise_min) {
Result = APSInt(std::min(Elem1, Elem2),
Call->getType()->isUnsignedIntegerOrEnumerationType());
} else {
llvm_unreachable("Wrong builtin ID");
}
INT_TYPE_SWITCH_NO_BOOL(ElemT,
{ Dst.elem<T>(I) = static_cast<T>(Result); });
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_pmul(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &,
const APSInt &)>
Fn) {
assert(Call->getArg(0)->getType()->isVectorType() &&
Call->getArg(1)->getType()->isVectorType());
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
PrimType ElemT = *S.getContext().classify(VT->getElementType());
unsigned NumElems = VT->getNumElements();
const auto *DestVT = Call->getType()->castAs<VectorType>();
PrimType DestElemT = *S.getContext().classify(DestVT->getElementType());
bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
unsigned DstElem = 0;
for (unsigned I = 0; I != NumElems; I += 2) {
APSInt Result;
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
APSInt LoLHS = LHS.elem<T>(I).toAPSInt();
APSInt HiLHS = LHS.elem<T>(I + 1).toAPSInt();
APSInt LoRHS = RHS.elem<T>(I).toAPSInt();
APSInt HiRHS = RHS.elem<T>(I + 1).toAPSInt();
Result = APSInt(Fn(LoLHS, HiLHS, LoRHS, HiRHS), DestUnsigned);
});
INT_TYPE_SWITCH_NO_BOOL(DestElemT,
{ Dst.elem<T>(DstElem) = static_cast<T>(Result); });
++DstElem;
}
Dst.initializeAllElements();
return true;
}
static bool interp_builtin_horizontal_int_binop(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<APInt(const APSInt &, const APSInt &)> Fn) {
const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
PrimType ElemT = *S.getContext().classify(VT->getElementType());
bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
unsigned NumElts = VT->getNumElements();
unsigned EltBits = S.getASTContext().getIntWidth(VT->getElementType());
unsigned EltsPerLane = 128 / EltBits;
unsigned Lanes = NumElts * EltBits / 128;
unsigned DestIndex = 0;
for (unsigned Lane = 0; Lane < Lanes; ++Lane) {
unsigned LaneStart = Lane * EltsPerLane;
for (unsigned I = 0; I < EltsPerLane; I += 2) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
APSInt Elem1 = LHS.elem<T>(LaneStart + I).toAPSInt();
APSInt Elem2 = LHS.elem<T>(LaneStart + I + 1).toAPSInt();
APSInt ResL = APSInt(Fn(Elem1, Elem2), DestUnsigned);
Dst.elem<T>(DestIndex++) = static_cast<T>(ResL);
});
}
for (unsigned I = 0; I < EltsPerLane; I += 2) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
APSInt Elem1 = RHS.elem<T>(LaneStart + I).toAPSInt();
APSInt Elem2 = RHS.elem<T>(LaneStart + I + 1).toAPSInt();
APSInt ResR = APSInt(Fn(Elem1, Elem2), DestUnsigned);
Dst.elem<T>(DestIndex++) = static_cast<T>(ResR);
});
}
}
Dst.initializeAllElements();
return true;
}
static bool interp_builtin_horizontal_fp_binop(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<APFloat(const APFloat &, const APFloat &,
llvm::RoundingMode)>
Fn) {
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
llvm::RoundingMode RM = getRoundingMode(FPO);
const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
unsigned NumElts = VT->getNumElements();
unsigned EltBits = S.getASTContext().getTypeSize(VT->getElementType());
unsigned NumLanes = NumElts * EltBits / 128;
unsigned NumElemsPerLane = NumElts / NumLanes;
unsigned HalfElemsPerLane = NumElemsPerLane / 2;
for (unsigned L = 0; L != NumElts; L += NumElemsPerLane) {
using T = PrimConv<PT_Float>::T;
for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
APFloat Elem1 = LHS.elem<T>(L + (2 * E) + 0).getAPFloat();
APFloat Elem2 = LHS.elem<T>(L + (2 * E) + 1).getAPFloat();
Dst.elem<T>(L + E) = static_cast<T>(Fn(Elem1, Elem2, RM));
}
for (unsigned E = 0; E != HalfElemsPerLane; ++E) {
APFloat Elem1 = RHS.elem<T>(L + (2 * E) + 0).getAPFloat();
APFloat Elem2 = RHS.elem<T>(L + (2 * E) + 1).getAPFloat();
Dst.elem<T>(L + E + HalfElemsPerLane) =
static_cast<T>(Fn(Elem1, Elem2, RM));
}
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_addsub(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
// Addsub: alternates between subtraction and addition
// Result[i] = (i % 2 == 0) ? (a[i] - b[i]) : (a[i] + b[i])
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
llvm::RoundingMode RM = getRoundingMode(FPO);
const auto *VT = Call->getArg(0)->getType()->castAs<VectorType>();
unsigned NumElems = VT->getNumElements();
using T = PrimConv<PT_Float>::T;
for (unsigned I = 0; I != NumElems; ++I) {
APFloat LElem = LHS.elem<T>(I).getAPFloat();
APFloat RElem = RHS.elem<T>(I).getAPFloat();
if (I % 2 == 0) {
// Even indices: subtract
LElem.subtract(RElem, RM);
} else {
// Odd indices: add
LElem.add(RElem, RM);
}
Dst.elem<T>(I) = static_cast<T>(LElem);
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_elementwise_triop_fp(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<APFloat(const APFloat &, const APFloat &,
const APFloat &, llvm::RoundingMode)>
Fn) {
assert(Call->getNumArgs() == 3);
FPOptions FPO = Call->getFPFeaturesInEffect(S.Ctx.getLangOpts());
llvm::RoundingMode RM = getRoundingMode(FPO);
QualType Arg1Type = Call->getArg(0)->getType();
QualType Arg2Type = Call->getArg(1)->getType();
QualType Arg3Type = Call->getArg(2)->getType();
// Non-vector floating point types.
if (!Arg1Type->isVectorType()) {
assert(!Arg2Type->isVectorType());
assert(!Arg3Type->isVectorType());
(void)Arg2Type;
(void)Arg3Type;
const Floating &Z = S.Stk.pop<Floating>();
const Floating &Y = S.Stk.pop<Floating>();
const Floating &X = S.Stk.pop<Floating>();
APFloat F = Fn(X.getAPFloat(), Y.getAPFloat(), Z.getAPFloat(), RM);
Floating Result = S.allocFloat(X.getSemantics());
Result.copy(F);
S.Stk.push<Floating>(Result);
return true;
}
// Vector type.
assert(Arg1Type->isVectorType() && Arg2Type->isVectorType() &&
Arg3Type->isVectorType());
const VectorType *VecTy = Arg1Type->castAs<VectorType>();
QualType ElemQT = VecTy->getElementType();
unsigned NumElems = VecTy->getNumElements();
assert(ElemQT == Arg2Type->castAs<VectorType>()->getElementType() &&
ElemQT == Arg3Type->castAs<VectorType>()->getElementType());
assert(NumElems == Arg2Type->castAs<VectorType>()->getNumElements() &&
NumElems == Arg3Type->castAs<VectorType>()->getNumElements());
assert(ElemQT->isRealFloatingType());
(void)ElemQT;
const Pointer &VZ = S.Stk.pop<Pointer>();
const Pointer &VY = S.Stk.pop<Pointer>();
const Pointer &VX = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
for (unsigned I = 0; I != NumElems; ++I) {
using T = PrimConv<PT_Float>::T;
APFloat X = VX.elem<T>(I).getAPFloat();
APFloat Y = VY.elem<T>(I).getAPFloat();
APFloat Z = VZ.elem<T>(I).getAPFloat();
APFloat F = Fn(X, Y, Z, RM);
Dst.elem<Floating>(I) = Floating(F);
}
Dst.initializeAllElements();
return true;
}
/// AVX512 predicated move: "Result = Mask[] ? LHS[] : RHS[]".
static bool interp__builtin_select(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
APSInt Mask = popToAPSInt(S, Call->getArg(0));
const Pointer &Dst = S.Stk.peek<Pointer>();
assert(LHS.getNumElems() == RHS.getNumElems());
assert(LHS.getNumElems() == Dst.getNumElems());
unsigned NumElems = LHS.getNumElems();
PrimType ElemT = LHS.getFieldDesc()->getPrimType();
PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
for (unsigned I = 0; I != NumElems; ++I) {
if (ElemT == PT_Float) {
assert(DstElemT == PT_Float);
Dst.elem<Floating>(I) =
Mask[I] ? LHS.elem<Floating>(I) : RHS.elem<Floating>(I);
} else {
APSInt Elem;
INT_TYPE_SWITCH(ElemT, {
Elem = Mask[I] ? LHS.elem<T>(I).toAPSInt() : RHS.elem<T>(I).toAPSInt();
});
INT_TYPE_SWITCH_NO_BOOL(DstElemT,
{ Dst.elem<T>(I) = static_cast<T>(Elem); });
}
}
Dst.initializeAllElements();
return true;
}
/// Scalar variant of AVX512 predicated select:
/// Result[i] = (Mask bit 0) ? LHS[i] : RHS[i], but only element 0 may change.
/// All other elements are taken from RHS.
static bool interp__builtin_select_scalar(InterpState &S,
const CallExpr *Call) {
unsigned N =
Call->getArg(1)->getType()->getAs<VectorType>()->getNumElements();
const Pointer &W = S.Stk.pop<Pointer>();
const Pointer &A = S.Stk.pop<Pointer>();
APSInt U = popToAPSInt(S, Call->getArg(0));
const Pointer &Dst = S.Stk.peek<Pointer>();
bool TakeA0 = U.getZExtValue() & 1ULL;
for (unsigned I = TakeA0; I != N; ++I)
Dst.elem<Floating>(I) = W.elem<Floating>(I);
if (TakeA0)
Dst.elem<Floating>(0) = A.elem<Floating>(0);
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_blend(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
APSInt Mask = popToAPSInt(S, Call->getArg(2));
const Pointer &TrueVec = S.Stk.pop<Pointer>();
const Pointer &FalseVec = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
assert(FalseVec.getNumElems() == TrueVec.getNumElems());
assert(FalseVec.getNumElems() == Dst.getNumElems());
unsigned NumElems = FalseVec.getNumElems();
PrimType ElemT = FalseVec.getFieldDesc()->getPrimType();
PrimType DstElemT = Dst.getFieldDesc()->getPrimType();
for (unsigned I = 0; I != NumElems; ++I) {
bool MaskBit = Mask[I % 8];
if (ElemT == PT_Float) {
assert(DstElemT == PT_Float);
Dst.elem<Floating>(I) =
MaskBit ? TrueVec.elem<Floating>(I) : FalseVec.elem<Floating>(I);
} else {
assert(DstElemT == ElemT);
INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
Dst.elem<T>(I) =
static_cast<T>(MaskBit ? TrueVec.elem<T>(I).toAPSInt()
: FalseVec.elem<T>(I).toAPSInt());
});
}
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_test_op(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<bool(const APInt &A, const APInt &B)> Fn) {
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
assert(LHS.getNumElems() == RHS.getNumElems());
unsigned SourceLen = LHS.getNumElems();
QualType ElemQT = getElemType(LHS);
OptPrimType ElemPT = S.getContext().classify(ElemQT);
unsigned LaneWidth = S.getASTContext().getTypeSize(ElemQT);
APInt AWide(LaneWidth * SourceLen, 0);
APInt BWide(LaneWidth * SourceLen, 0);
for (unsigned I = 0; I != SourceLen; ++I) {
APInt ALane;
APInt BLane;
if (ElemQT->isIntegerType()) { // Get value.
INT_TYPE_SWITCH_NO_BOOL(*ElemPT, {
ALane = LHS.elem<T>(I).toAPSInt();
BLane = RHS.elem<T>(I).toAPSInt();
});
} else if (ElemQT->isFloatingType()) { // Get only sign bit.
using T = PrimConv<PT_Float>::T;
ALane = LHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
BLane = RHS.elem<T>(I).getAPFloat().bitcastToAPInt().isNegative();
} else { // Must be integer or floating type.
return false;
}
AWide.insertBits(ALane, I * LaneWidth);
BWide.insertBits(BLane, I * LaneWidth);
}
pushInteger(S, Fn(AWide, BWide), Call->getType());
return true;
}
static bool interp__builtin_ia32_movmsk_op(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
assert(Call->getNumArgs() == 1);
const Pointer &Source = S.Stk.pop<Pointer>();
unsigned SourceLen = Source.getNumElems();
QualType ElemQT = getElemType(Source);
OptPrimType ElemT = S.getContext().classify(ElemQT);
unsigned ResultLen =
S.getASTContext().getTypeSize(Call->getType()); // Always 32-bit integer.
APInt Result(ResultLen, 0);
for (unsigned I = 0; I != SourceLen; ++I) {
APInt Elem;
if (ElemQT->isIntegerType()) {
INT_TYPE_SWITCH_NO_BOOL(*ElemT, { Elem = Source.elem<T>(I).toAPSInt(); });
} else if (ElemQT->isRealFloatingType()) {
using T = PrimConv<PT_Float>::T;
Elem = Source.elem<T>(I).getAPFloat().bitcastToAPInt();
} else {
return false;
}
Result.setBitVal(I, Elem.isNegative());
}
pushInteger(S, Result, Call->getType());
return true;
}
static bool interp__builtin_elementwise_triop(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<APInt(const APSInt &, const APSInt &, const APSInt &)>
Fn) {
assert(Call->getNumArgs() == 3);
QualType Arg0Type = Call->getArg(0)->getType();
QualType Arg2Type = Call->getArg(2)->getType();
// Non-vector integer types.
if (!Arg0Type->isVectorType()) {
const APSInt &Op2 = popToAPSInt(S, Arg2Type);
const APSInt &Op1 = popToAPSInt(S, Call->getArg(1));
const APSInt &Op0 = popToAPSInt(S, Arg0Type);
APSInt Result = APSInt(Fn(Op0, Op1, Op2), Op0.isUnsigned());
pushInteger(S, Result, Call->getType());
return true;
}
const auto *VecT = Arg0Type->castAs<VectorType>();
PrimType ElemT = *S.getContext().classify(VecT->getElementType());
unsigned NumElems = VecT->getNumElements();
bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
// Vector + Vector + Scalar case.
if (!Arg2Type->isVectorType()) {
APSInt Op2 = popToAPSInt(S, Arg2Type);
const Pointer &Op1 = S.Stk.pop<Pointer>();
const Pointer &Op0 = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
for (unsigned I = 0; I != NumElems; ++I) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
Dst.elem<T>(I) = static_cast<T>(APSInt(
Fn(Op0.elem<T>(I).toAPSInt(), Op1.elem<T>(I).toAPSInt(), Op2),
DestUnsigned));
});
}
Dst.initializeAllElements();
return true;
}
// Vector type.
const Pointer &Op2 = S.Stk.pop<Pointer>();
const Pointer &Op1 = S.Stk.pop<Pointer>();
const Pointer &Op0 = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
for (unsigned I = 0; I != NumElems; ++I) {
APSInt Val0, Val1, Val2;
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
Val0 = Op0.elem<T>(I).toAPSInt();
Val1 = Op1.elem<T>(I).toAPSInt();
Val2 = Op2.elem<T>(I).toAPSInt();
});
APSInt Result = APSInt(Fn(Val0, Val1, Val2), Val0.isUnsigned());
INT_TYPE_SWITCH_NO_BOOL(ElemT,
{ Dst.elem<T>(I) = static_cast<T>(Result); });
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_x86_extract_vector(InterpState &S, CodePtr OpPC,
const CallExpr *Call,
unsigned ID) {
assert(Call->getNumArgs() == 2);
APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));
uint64_t Index = ImmAPS.getZExtValue();
const Pointer &Src = S.Stk.pop<Pointer>();
if (!Src.getFieldDesc()->isPrimitiveArray())
return false;
const Pointer &Dst = S.Stk.peek<Pointer>();
if (!Dst.getFieldDesc()->isPrimitiveArray())
return false;
unsigned SrcElems = Src.getNumElems();
unsigned DstElems = Dst.getNumElems();
unsigned NumLanes = SrcElems / DstElems;
unsigned Lane = static_cast<unsigned>(Index % NumLanes);
unsigned ExtractPos = Lane * DstElems;
PrimType ElemT = Src.getFieldDesc()->getPrimType();
TYPE_SWITCH(ElemT, {
for (unsigned I = 0; I != DstElems; ++I) {
Dst.elem<T>(I) = Src.elem<T>(ExtractPos + I);
}
});
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_x86_extract_vector_masked(InterpState &S,
CodePtr OpPC,
const CallExpr *Call,
unsigned ID) {
assert(Call->getNumArgs() == 4);
APSInt MaskAPS = popToAPSInt(S, Call->getArg(3));
const Pointer &Merge = S.Stk.pop<Pointer>();
APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));
const Pointer &Src = S.Stk.pop<Pointer>();
if (!Src.getFieldDesc()->isPrimitiveArray() ||
!Merge.getFieldDesc()->isPrimitiveArray())
return false;
const Pointer &Dst = S.Stk.peek<Pointer>();
if (!Dst.getFieldDesc()->isPrimitiveArray())
return false;
unsigned SrcElems = Src.getNumElems();
unsigned DstElems = Dst.getNumElems();
unsigned NumLanes = SrcElems / DstElems;
unsigned Lane = static_cast<unsigned>(ImmAPS.getZExtValue() % NumLanes);
unsigned Base = Lane * DstElems;
PrimType ElemT = Src.getFieldDesc()->getPrimType();
TYPE_SWITCH(ElemT, {
for (unsigned I = 0; I != DstElems; ++I) {
if (MaskAPS[I])
Dst.elem<T>(I) = Src.elem<T>(Base + I);
else
Dst.elem<T>(I) = Merge.elem<T>(I);
}
});
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_x86_insert_subvector(InterpState &S, CodePtr OpPC,
const CallExpr *Call,
unsigned ID) {
assert(Call->getNumArgs() == 3);
APSInt ImmAPS = popToAPSInt(S, Call->getArg(2));
uint64_t Index = ImmAPS.getZExtValue();
const Pointer &SubVec = S.Stk.pop<Pointer>();
if (!SubVec.getFieldDesc()->isPrimitiveArray())
return false;
const Pointer &BaseVec = S.Stk.pop<Pointer>();
if (!BaseVec.getFieldDesc()->isPrimitiveArray())
return false;
const Pointer &Dst = S.Stk.peek<Pointer>();
unsigned BaseElements = BaseVec.getNumElems();
unsigned SubElements = SubVec.getNumElems();
assert(SubElements != 0 && BaseElements != 0 &&
(BaseElements % SubElements) == 0);
unsigned NumLanes = BaseElements / SubElements;
unsigned Lane = static_cast<unsigned>(Index % NumLanes);
unsigned InsertPos = Lane * SubElements;
PrimType ElemT = BaseVec.getFieldDesc()->getPrimType();
TYPE_SWITCH(ElemT, {
for (unsigned I = 0; I != BaseElements; ++I)
Dst.elem<T>(I) = BaseVec.elem<T>(I);
for (unsigned I = 0; I != SubElements; ++I)
Dst.elem<T>(InsertPos + I) = SubVec.elem<T>(I);
});
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_phminposuw(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
assert(Call->getNumArgs() == 1);
const Pointer &Source = S.Stk.pop<Pointer>();
const Pointer &Dest = S.Stk.peek<Pointer>();
unsigned SourceLen = Source.getNumElems();
QualType ElemQT = getElemType(Source);
OptPrimType ElemT = S.getContext().classify(ElemQT);
unsigned ElemBitWidth = S.getASTContext().getTypeSize(ElemQT);
bool DestUnsigned = Call->getCallReturnType(S.getASTContext())
->castAs<VectorType>()
->getElementType()
->isUnsignedIntegerOrEnumerationType();
INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
APSInt MinIndex(ElemBitWidth, DestUnsigned);
APSInt MinVal = Source.elem<T>(0).toAPSInt();
for (unsigned I = 1; I != SourceLen; ++I) {
APSInt Val = Source.elem<T>(I).toAPSInt();
if (MinVal.ugt(Val)) {
MinVal = Val;
MinIndex = I;
}
}
Dest.elem<T>(0) = static_cast<T>(MinVal);
Dest.elem<T>(1) = static_cast<T>(MinIndex);
for (unsigned I = 2; I != SourceLen; ++I) {
Dest.elem<T>(I) = static_cast<T>(APSInt(ElemBitWidth, DestUnsigned));
}
});
Dest.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_pternlog(InterpState &S, CodePtr OpPC,
const CallExpr *Call, bool MaskZ) {
assert(Call->getNumArgs() == 5);
APInt U = popToAPSInt(S, Call->getArg(4)); // Lane mask
APInt Imm = popToAPSInt(S, Call->getArg(3)); // Ternary truth table
const Pointer &C = S.Stk.pop<Pointer>();
const Pointer &B = S.Stk.pop<Pointer>();
const Pointer &A = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
unsigned DstLen = A.getNumElems();
QualType ElemQT = getElemType(A);
OptPrimType ElemT = S.getContext().classify(ElemQT);
unsigned LaneWidth = S.getASTContext().getTypeSize(ElemQT);
bool DstUnsigned = ElemQT->isUnsignedIntegerOrEnumerationType();
INT_TYPE_SWITCH_NO_BOOL(*ElemT, {
for (unsigned I = 0; I != DstLen; ++I) {
APInt ALane = A.elem<T>(I).toAPSInt();
APInt BLane = B.elem<T>(I).toAPSInt();
APInt CLane = C.elem<T>(I).toAPSInt();
APInt RLane(LaneWidth, 0);
if (U[I]) { // If lane not masked, compute ternary logic.
for (unsigned Bit = 0; Bit != LaneWidth; ++Bit) {
unsigned ABit = ALane[Bit];
unsigned BBit = BLane[Bit];
unsigned CBit = CLane[Bit];
unsigned Idx = (ABit << 2) | (BBit << 1) | (CBit);
RLane.setBitVal(Bit, Imm[Idx]);
}
Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
} else if (MaskZ) { // If zero masked, zero the lane.
Dst.elem<T>(I) = static_cast<T>(APSInt(RLane, DstUnsigned));
} else { // Just masked, put in A lane.
Dst.elem<T>(I) = static_cast<T>(APSInt(ALane, DstUnsigned));
}
}
});
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_vec_ext(InterpState &S, CodePtr OpPC,
const CallExpr *Call, unsigned ID) {
assert(Call->getNumArgs() == 2);
APSInt ImmAPS = popToAPSInt(S, Call->getArg(1));
const Pointer &Vec = S.Stk.pop<Pointer>();
if (!Vec.getFieldDesc()->isPrimitiveArray())
return false;
unsigned NumElems = Vec.getNumElems();
unsigned Index =
static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
PrimType ElemT = Vec.getFieldDesc()->getPrimType();
// FIXME(#161685): Replace float+int split with a numeric-only type switch
if (ElemT == PT_Float) {
S.Stk.push<Floating>(Vec.elem<Floating>(Index));
return true;
}
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
APSInt V = Vec.elem<T>(Index).toAPSInt();
pushInteger(S, V, Call->getType());
});
return true;
}
static bool interp__builtin_vec_set(InterpState &S, CodePtr OpPC,
const CallExpr *Call, unsigned ID) {
assert(Call->getNumArgs() == 3);
APSInt ImmAPS = popToAPSInt(S, Call->getArg(2));
APSInt ValAPS = popToAPSInt(S, Call->getArg(1));
const Pointer &Base = S.Stk.pop<Pointer>();
if (!Base.getFieldDesc()->isPrimitiveArray())
return false;
const Pointer &Dst = S.Stk.peek<Pointer>();
unsigned NumElems = Base.getNumElems();
unsigned Index =
static_cast<unsigned>(ImmAPS.getZExtValue() & (NumElems - 1));
PrimType ElemT = Base.getFieldDesc()->getPrimType();
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
for (unsigned I = 0; I != NumElems; ++I)
Dst.elem<T>(I) = Base.elem<T>(I);
Dst.elem<T>(Index) = static_cast<T>(ValAPS);
});
Dst.initializeAllElements();
return true;
}
static bool evalICmpImm(uint8_t Imm, const APSInt &A, const APSInt &B,
bool IsUnsigned) {
switch (Imm & 0x7) {
case 0x00: // _MM_CMPINT_EQ
return (A == B);
case 0x01: // _MM_CMPINT_LT
return IsUnsigned ? A.ult(B) : A.slt(B);
case 0x02: // _MM_CMPINT_LE
return IsUnsigned ? A.ule(B) : A.sle(B);
case 0x03: // _MM_CMPINT_FALSE
return false;
case 0x04: // _MM_CMPINT_NE
return (A != B);
case 0x05: // _MM_CMPINT_NLT
return IsUnsigned ? A.ugt(B) : A.sgt(B);
case 0x06: // _MM_CMPINT_NLE
return IsUnsigned ? A.uge(B) : A.sge(B);
case 0x07: // _MM_CMPINT_TRUE
return true;
default:
llvm_unreachable("Invalid Op");
}
}
static bool interp__builtin_ia32_cmp_mask(InterpState &S, CodePtr OpPC,
const CallExpr *Call, unsigned ID,
bool IsUnsigned) {
assert(Call->getNumArgs() == 4);
APSInt Mask = popToAPSInt(S, Call->getArg(3));
APSInt Opcode = popToAPSInt(S, Call->getArg(2));
unsigned CmpOp = static_cast<unsigned>(Opcode.getZExtValue());
const Pointer &RHS = S.Stk.pop<Pointer>();
const Pointer &LHS = S.Stk.pop<Pointer>();
assert(LHS.getNumElems() == RHS.getNumElems());
APInt RetMask = APInt::getZero(LHS.getNumElems());
unsigned VectorLen = LHS.getNumElems();
PrimType ElemT = LHS.getFieldDesc()->getPrimType();
for (unsigned ElemNum = 0; ElemNum < VectorLen; ++ElemNum) {
APSInt A, B;
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
A = LHS.elem<T>(ElemNum).toAPSInt();
B = RHS.elem<T>(ElemNum).toAPSInt();
});
RetMask.setBitVal(ElemNum,
Mask[ElemNum] && evalICmpImm(CmpOp, A, B, IsUnsigned));
}
pushInteger(S, RetMask, Call->getType());
return true;
}
static bool interp__builtin_ia32_vpconflict(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
assert(Call->getNumArgs() == 1);
QualType Arg0Type = Call->getArg(0)->getType();
const auto *VecT = Arg0Type->castAs<VectorType>();
PrimType ElemT = *S.getContext().classify(VecT->getElementType());
unsigned NumElems = VecT->getNumElements();
bool DestUnsigned = Call->getType()->isUnsignedIntegerOrEnumerationType();
const Pointer &Src = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
for (unsigned I = 0; I != NumElems; ++I) {
INT_TYPE_SWITCH_NO_BOOL(ElemT, {
APSInt ElemI = Src.elem<T>(I).toAPSInt();
APInt ConflictMask(ElemI.getBitWidth(), 0);
for (unsigned J = 0; J != I; ++J) {
APSInt ElemJ = Src.elem<T>(J).toAPSInt();
ConflictMask.setBitVal(J, ElemI == ElemJ);
}
Dst.elem<T>(I) = static_cast<T>(APSInt(ConflictMask, DestUnsigned));
});
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_cvt_vec2mask(InterpState &S, CodePtr OpPC,
const CallExpr *Call,
unsigned ID) {
assert(Call->getNumArgs() == 1);
const Pointer &Vec = S.Stk.pop<Pointer>();
unsigned RetWidth = S.getASTContext().getIntWidth(Call->getType());
APInt RetMask(RetWidth, 0);
unsigned VectorLen = Vec.getNumElems();
PrimType ElemT = Vec.getFieldDesc()->getPrimType();
for (unsigned ElemNum = 0; ElemNum != VectorLen; ++ElemNum) {
APSInt A;
INT_TYPE_SWITCH_NO_BOOL(ElemT, { A = Vec.elem<T>(ElemNum).toAPSInt(); });
unsigned MSB = A[A.getBitWidth() - 1];
RetMask.setBitVal(ElemNum, MSB);
}
pushInteger(S, RetMask, Call->getType());
return true;
}
static bool interp__builtin_ia32_shuffle_generic(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<std::pair<unsigned, int>(unsigned, unsigned)>
GetSourceIndex) {
assert(Call->getNumArgs() == 2 || Call->getNumArgs() == 3);
unsigned ShuffleMask = 0;
Pointer A, MaskVector, B;
bool IsVectorMask = false;
bool IsSingleOperand = (Call->getNumArgs() == 2);
if (IsSingleOperand) {
QualType MaskType = Call->getArg(1)->getType();
if (MaskType->isVectorType()) {
IsVectorMask = true;
MaskVector = S.Stk.pop<Pointer>();
A = S.Stk.pop<Pointer>();
B = A;
} else if (MaskType->isIntegerType()) {
ShuffleMask = popToAPSInt(S, Call->getArg(1)).getZExtValue();
A = S.Stk.pop<Pointer>();
B = A;
} else {
return false;
}
} else {
QualType Arg2Type = Call->getArg(2)->getType();
if (Arg2Type->isVectorType()) {
IsVectorMask = true;
B = S.Stk.pop<Pointer>();
MaskVector = S.Stk.pop<Pointer>();
A = S.Stk.pop<Pointer>();
} else if (Arg2Type->isIntegerType()) {
ShuffleMask = popToAPSInt(S, Call->getArg(2)).getZExtValue();
B = S.Stk.pop<Pointer>();
A = S.Stk.pop<Pointer>();
} else {
return false;
}
}
QualType Arg0Type = Call->getArg(0)->getType();
const auto *VecT = Arg0Type->castAs<VectorType>();
PrimType ElemT = *S.getContext().classify(VecT->getElementType());
unsigned NumElems = VecT->getNumElements();
const Pointer &Dst = S.Stk.peek<Pointer>();
PrimType MaskElemT = PT_Uint32;
if (IsVectorMask) {
QualType Arg1Type = Call->getArg(1)->getType();
const auto *MaskVecT = Arg1Type->castAs<VectorType>();
QualType MaskElemType = MaskVecT->getElementType();
MaskElemT = *S.getContext().classify(MaskElemType);
}
for (unsigned DstIdx = 0; DstIdx != NumElems; ++DstIdx) {
if (IsVectorMask) {
INT_TYPE_SWITCH(MaskElemT, {
ShuffleMask = static_cast<unsigned>(MaskVector.elem<T>(DstIdx));
});
}
auto [SrcVecIdx, SrcIdx] = GetSourceIndex(DstIdx, ShuffleMask);
if (SrcIdx < 0) {
// Zero out this element
if (ElemT == PT_Float) {
Dst.elem<Floating>(DstIdx) = Floating(
S.getASTContext().getFloatTypeSemantics(VecT->getElementType()));
} else {
INT_TYPE_SWITCH_NO_BOOL(ElemT, { Dst.elem<T>(DstIdx) = T::from(0); });
}
} else {
const Pointer &Src = (SrcVecIdx == 0) ? A : B;
TYPE_SWITCH(ElemT, { Dst.elem<T>(DstIdx) = Src.elem<T>(SrcIdx); });
}
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_shift_with_count(
InterpState &S, CodePtr OpPC, const CallExpr *Call,
llvm::function_ref<APInt(const APInt &, uint64_t)> ShiftOp,
llvm::function_ref<APInt(const APInt &, unsigned)> OverflowOp) {
assert(Call->getNumArgs() == 2);
const Pointer &Count = S.Stk.pop<Pointer>();
const Pointer &Source = S.Stk.pop<Pointer>();
QualType SourceType = Call->getArg(0)->getType();
QualType CountType = Call->getArg(1)->getType();
assert(SourceType->isVectorType() && CountType->isVectorType());
const auto *SourceVecT = SourceType->castAs<VectorType>();
const auto *CountVecT = CountType->castAs<VectorType>();
PrimType SourceElemT = *S.getContext().classify(SourceVecT->getElementType());
PrimType CountElemT = *S.getContext().classify(CountVecT->getElementType());
const Pointer &Dst = S.Stk.peek<Pointer>();
unsigned DestEltWidth =
S.getASTContext().getTypeSize(SourceVecT->getElementType());
bool IsDestUnsigned = SourceVecT->getElementType()->isUnsignedIntegerType();
unsigned DestLen = SourceVecT->getNumElements();
unsigned CountEltWidth =
S.getASTContext().getTypeSize(CountVecT->getElementType());
unsigned NumBitsInQWord = 64;
unsigned NumCountElts = NumBitsInQWord / CountEltWidth;
uint64_t CountLQWord = 0;
for (unsigned EltIdx = 0; EltIdx != NumCountElts; ++EltIdx) {
uint64_t Elt = 0;
INT_TYPE_SWITCH(CountElemT,
{ Elt = static_cast<uint64_t>(Count.elem<T>(EltIdx)); });
CountLQWord |= (Elt << (EltIdx * CountEltWidth));
}
for (unsigned EltIdx = 0; EltIdx != DestLen; ++EltIdx) {
APSInt Elt;
INT_TYPE_SWITCH(SourceElemT, { Elt = Source.elem<T>(EltIdx).toAPSInt(); });
APInt Result;
if (CountLQWord < DestEltWidth) {
Result = ShiftOp(Elt, CountLQWord);
} else {
Result = OverflowOp(Elt, DestEltWidth);
}
if (IsDestUnsigned) {
INT_TYPE_SWITCH(SourceElemT, {
Dst.elem<T>(EltIdx) = T::from(Result.getZExtValue());
});
} else {
INT_TYPE_SWITCH(SourceElemT, {
Dst.elem<T>(EltIdx) = T::from(Result.getSExtValue());
});
}
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_shufbitqmb_mask(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
assert(Call->getNumArgs() == 3);
QualType SourceType = Call->getArg(0)->getType();
QualType ShuffleMaskType = Call->getArg(1)->getType();
QualType ZeroMaskType = Call->getArg(2)->getType();
if (!SourceType->isVectorType() || !ShuffleMaskType->isVectorType() ||
!ZeroMaskType->isIntegerType()) {
return false;
}
Pointer Source, ShuffleMask;
APSInt ZeroMask = popToAPSInt(S, Call->getArg(2));
ShuffleMask = S.Stk.pop<Pointer>();
Source = S.Stk.pop<Pointer>();
const auto *SourceVecT = SourceType->castAs<VectorType>();
const auto *ShuffleMaskVecT = ShuffleMaskType->castAs<VectorType>();
assert(SourceVecT->getNumElements() == ShuffleMaskVecT->getNumElements());
assert(ZeroMask.getBitWidth() == SourceVecT->getNumElements());
PrimType SourceElemT = *S.getContext().classify(SourceVecT->getElementType());
PrimType ShuffleMaskElemT =
*S.getContext().classify(ShuffleMaskVecT->getElementType());
unsigned NumBytesInQWord = 8;
unsigned NumBitsInByte = 8;
unsigned NumBytes = SourceVecT->getNumElements();
unsigned NumQWords = NumBytes / NumBytesInQWord;
unsigned RetWidth = ZeroMask.getBitWidth();
APSInt RetMask(llvm::APInt(RetWidth, 0), /*isUnsigned=*/true);
for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
APInt SourceQWord(64, 0);
for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
uint64_t Byte = 0;
INT_TYPE_SWITCH(SourceElemT, {
Byte = static_cast<uint64_t>(
Source.elem<T>(QWordId * NumBytesInQWord + ByteIdx));
});
SourceQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
}
for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
unsigned SelIdx = QWordId * NumBytesInQWord + ByteIdx;
unsigned M = 0;
INT_TYPE_SWITCH(ShuffleMaskElemT, {
M = static_cast<unsigned>(ShuffleMask.elem<T>(SelIdx)) & 0x3F;
});
if (ZeroMask[SelIdx]) {
RetMask.setBitVal(SelIdx, SourceQWord[M]);
}
}
}
pushInteger(S, RetMask, Call->getType());
return true;
}
static bool interp__builtin_ia32_vcvtps2ph(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
// Arguments are: vector of floats, rounding immediate
assert(Call->getNumArgs() == 2);
APSInt Imm = popToAPSInt(S, Call->getArg(1));
const Pointer &Src = S.Stk.pop<Pointer>();
const Pointer &Dst = S.Stk.peek<Pointer>();
assert(Src.getFieldDesc()->isPrimitiveArray());
assert(Dst.getFieldDesc()->isPrimitiveArray());
const auto *SrcVTy = Call->getArg(0)->getType()->castAs<VectorType>();
unsigned SrcNumElems = SrcVTy->getNumElements();
const auto *DstVTy = Call->getType()->castAs<VectorType>();
unsigned DstNumElems = DstVTy->getNumElements();
const llvm::fltSemantics &HalfSem =
S.getASTContext().getFloatTypeSemantics(S.getASTContext().HalfTy);
// imm[2] == 1 means use MXCSR rounding mode.
// In that case, we can only evaluate if the conversion is exact.
int ImmVal = Imm.getZExtValue();
bool UseMXCSR = (ImmVal & 4) != 0;
bool IsFPConstrained =
Call->getFPFeaturesInEffect(S.getASTContext().getLangOpts())
.isFPConstrained();
llvm::RoundingMode RM;
if (!UseMXCSR) {
switch (ImmVal & 3) {
case 0:
RM = llvm::RoundingMode::NearestTiesToEven;
break;
case 1:
RM = llvm::RoundingMode::TowardNegative;
break;
case 2:
RM = llvm::RoundingMode::TowardPositive;
break;
case 3:
RM = llvm::RoundingMode::TowardZero;
break;
default:
llvm_unreachable("Invalid immediate rounding mode");
}
} else {
// For MXCSR, we must check for exactness. We can use any rounding mode
// for the trial conversion since the result is the same if it's exact.
RM = llvm::RoundingMode::NearestTiesToEven;
}
QualType DstElemQT = Dst.getFieldDesc()->getElemQualType();
PrimType DstElemT = *S.getContext().classify(DstElemQT);
for (unsigned I = 0; I != SrcNumElems; ++I) {
Floating SrcVal = Src.elem<Floating>(I);
APFloat DstVal = SrcVal.getAPFloat();
bool LostInfo;
APFloat::opStatus St = DstVal.convert(HalfSem, RM, &LostInfo);
if (UseMXCSR && IsFPConstrained && St != APFloat::opOK) {
S.FFDiag(S.Current->getSource(OpPC),
diag::note_constexpr_dynamic_rounding);
return false;
}
INT_TYPE_SWITCH_NO_BOOL(DstElemT, {
// Convert the destination value's bit pattern to an unsigned integer,
// then reconstruct the element using the target type's 'from' method.
uint64_t RawBits = DstVal.bitcastToAPInt().getZExtValue();
Dst.elem<T>(I) = T::from(RawBits);
});
}
// Zero out remaining elements if the destination has more elements
// (e.g., vcvtps2ph converting 4 floats to 8 shorts).
if (DstNumElems > SrcNumElems) {
for (unsigned I = SrcNumElems; I != DstNumElems; ++I) {
INT_TYPE_SWITCH_NO_BOOL(DstElemT, { Dst.elem<T>(I) = T::from(0); });
}
}
Dst.initializeAllElements();
return true;
}
static bool interp__builtin_ia32_multishiftqb(InterpState &S, CodePtr OpPC,
const CallExpr *Call) {
assert(Call->getNumArgs() == 2);
QualType ATy = Call->getArg(0)->getType();
QualType BTy = Call->getArg(1)->getType();
if (!ATy->isVectorType() || !BTy->isVectorType()) {
return false;
}
const Pointer &BPtr = S.Stk.pop<Pointer>();
const Pointer &APtr = S.Stk.pop<Pointer>();
const auto *AVecT = ATy->castAs<VectorType>();
assert(AVecT->getNumElements() ==
BTy->castAs<VectorType>()->getNumElements());
PrimType ElemT = *S.getContext().classify(AVecT->getElementType());
unsigned NumBytesInQWord = 8;
unsigned NumBitsInByte = 8;
unsigned NumBytes = AVecT->getNumElements();
unsigned NumQWords = NumBytes / NumBytesInQWord;
const Pointer &Dst = S.Stk.peek<Pointer>();
for (unsigned QWordId = 0; QWordId != NumQWords; ++QWordId) {
APInt BQWord(64, 0);
for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
INT_TYPE_SWITCH(ElemT, {
uint64_t Byte = static_cast<uint64_t>(BPtr.elem<T>(Idx));
BQWord.insertBits(APInt(8, Byte & 0xFF), ByteIdx * NumBitsInByte);
});
}
for (unsigned ByteIdx = 0; ByteIdx != NumBytesInQWord; ++ByteIdx) {
unsigned Idx = QWordId * NumBytesInQWord + ByteIdx;
uint64_t Ctrl = 0;
INT_TYPE_SWITCH(
ElemT, { Ctrl = static_cast<uint64_t>(APtr.elem<T>(Idx)) & 0x3F; });
APInt Byte(8, 0);
for (unsigned BitIdx = 0; BitIdx != NumBitsInByte; ++BitIdx) {
Byte.setBitVal(BitIdx, BQWord[(Ctrl + BitIdx) & 0x3F]);
}
INT_TYPE_SWITCH(ElemT,
{ Dst.elem<T>(Idx) = T::from(Byte.getZExtValue()); });
}
}
Dst.initializeAllElements();
return true;
}
bool InterpretBuiltin(InterpState &S, CodePtr OpPC, const CallExpr *Call,
uint32_t BuiltinID) {
if (!S.getASTContext().BuiltinInfo.isConstantEvaluated(BuiltinID))
return Invalid(S, OpPC);
const InterpFrame *Frame = S.Current;
switch (BuiltinID) {
case Builtin::BI__builtin_is_constant_evaluated:
return interp__builtin_is_constant_evaluated(S, OpPC, Frame, Call);
case Builtin::BI__builtin_assume:
case Builtin::BI__assume:
return interp__builtin_assume(S, OpPC, Frame, Call);
case Builtin::BI__builtin_strcmp:
case Builtin::BIstrcmp:
case Builtin::BI__builtin_strncmp:
case Builtin::BIstrncmp:
case Builtin::BI__builtin_wcsncmp:
case Builtin::BIwcsncmp:
case Builtin::BI__builtin_wcscmp:
case Builtin::BIwcscmp:
return interp__builtin_strcmp(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__builtin_strlen:
case Builtin::BIstrlen:
case Builtin::BI__builtin_wcslen:
case Builtin::BIwcslen:
return interp__builtin_strlen(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__builtin_nan:
case Builtin::BI__builtin_nanf:
case Builtin::BI__builtin_nanl:
case Builtin::BI__builtin_nanf16:
case Builtin::BI__builtin_nanf128:
return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/false);
case Builtin::BI__builtin_nans:
case Builtin::BI__builtin_nansf:
case Builtin::BI__builtin_nansl:
case Builtin::BI__builtin_nansf16:
case Builtin::BI__builtin_nansf128:
return interp__builtin_nan(S, OpPC, Frame, Call, /*Signaling=*/true);
case Builtin::BI__builtin_huge_val:
case Builtin::BI__builtin_huge_valf:
case Builtin::BI__builtin_huge_vall:
case Builtin::BI__builtin_huge_valf16:
case Builtin::BI__builtin_huge_valf128:
case Builtin::BI__builtin_inf:
case Builtin::BI__builtin_inff:
case Builtin::BI__builtin_infl:
case Builtin::BI__builtin_inff16:
case Builtin::BI__builtin_inff128:
return interp__builtin_inf(S, OpPC, Frame, Call);
case Builtin::BI__builtin_copysign:
case Builtin::BI__builtin_copysignf:
case Builtin::BI__builtin_copysignl:
case Builtin::BI__builtin_copysignf128:
return interp__builtin_copysign(S, OpPC, Frame);
case Builtin::BI__builtin_fmin:
case Builtin::BI__builtin_fminf:
case Builtin::BI__builtin_fminl:
case Builtin::BI__builtin_fminf16:
case Builtin::BI__builtin_fminf128:
return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/false);
case Builtin::BI__builtin_fminimum_num:
case Builtin::BI__builtin_fminimum_numf:
case Builtin::BI__builtin_fminimum_numl:
case Builtin::BI__builtin_fminimum_numf16:
case Builtin::BI__builtin_fminimum_numf128:
return interp__builtin_fmin(S, OpPC, Frame, /*IsNumBuiltin=*/true);
case Builtin::BI__builtin_fmax:
case Builtin::BI__builtin_fmaxf:
case Builtin::BI__builtin_fmaxl:
case Builtin::BI__builtin_fmaxf16:
case Builtin::BI__builtin_fmaxf128:
return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/false);
case Builtin::BI__builtin_fmaximum_num:
case Builtin::BI__builtin_fmaximum_numf:
case Builtin::BI__builtin_fmaximum_numl:
case Builtin::BI__builtin_fmaximum_numf16:
case Builtin::BI__builtin_fmaximum_numf128:
return interp__builtin_fmax(S, OpPC, Frame, /*IsNumBuiltin=*/true);
case Builtin::BI__builtin_isnan:
return interp__builtin_isnan(S, OpPC, Frame, Call);
case Builtin::BI__builtin_issignaling:
return interp__builtin_issignaling(S, OpPC, Frame, Call);
case Builtin::BI__builtin_isinf:
return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/false, Call);
case Builtin::BI__builtin_isinf_sign:
return interp__builtin_isinf(S, OpPC, Frame, /*Sign=*/true, Call);
case Builtin::BI__builtin_isfinite:
return interp__builtin_isfinite(S, OpPC, Frame, Call);
case Builtin::BI__builtin_isnormal:
return interp__builtin_isnormal(S, OpPC, Frame, Call);
case Builtin::BI__builtin_issubnormal:
return interp__builtin_issubnormal(S, OpPC, Frame, Call);
case Builtin::BI__builtin_iszero:
return interp__builtin_iszero(S, OpPC, Frame, Call);
case Builtin::BI__builtin_signbit:
case Builtin::BI__builtin_signbitf:
case Builtin::BI__builtin_signbitl:
return interp__builtin_signbit(S, OpPC, Frame, Call);
case Builtin::BI__builtin_isgreater:
case Builtin::BI__builtin_isgreaterequal:
case Builtin::BI__builtin_isless:
case Builtin::BI__builtin_islessequal:
case Builtin::BI__builtin_islessgreater:
case Builtin::BI__builtin_isunordered:
return interp_floating_comparison(S, OpPC, Call, BuiltinID);
case Builtin::BI__builtin_isfpclass:
return interp__builtin_isfpclass(S, OpPC, Frame, Call);
case Builtin::BI__builtin_fpclassify:
return interp__builtin_fpclassify(S, OpPC, Frame, Call);
case Builtin::BI__builtin_fabs:
case Builtin::BI__builtin_fabsf:
case Builtin::BI__builtin_fabsl:
case Builtin::BI__builtin_fabsf128:
return interp__builtin_fabs(S, OpPC, Frame);
case Builtin::BI__builtin_abs:
case Builtin::BI__builtin_labs:
case Builtin::BI__builtin_llabs:
return interp__builtin_abs(S, OpPC, Frame, Call);
case Builtin::BI__builtin_popcount:
case Builtin::BI__builtin_popcountl:
case Builtin::BI__builtin_popcountll:
case Builtin::BI__builtin_popcountg:
case Builtin::BI__popcnt16: // Microsoft variants of popcount
case Builtin::BI__popcnt:
case Builtin::BI__popcnt64:
return interp__builtin_popcount(S, OpPC, Frame, Call);
case Builtin::BI__builtin_parity:
case Builtin::BI__builtin_parityl:
case Builtin::BI__builtin_parityll:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Val) {
return APInt(Val.getBitWidth(), Val.popcount() % 2);
});
case Builtin::BI__builtin_clrsb:
case Builtin::BI__builtin_clrsbl:
case Builtin::BI__builtin_clrsbll:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Val) {
return APInt(Val.getBitWidth(),
Val.getBitWidth() - Val.getSignificantBits());
});
case Builtin::BI__builtin_bitreverse8:
case Builtin::BI__builtin_bitreverse16:
case Builtin::BI__builtin_bitreverse32:
case Builtin::BI__builtin_bitreverse64:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Val) { return Val.reverseBits(); });
case Builtin::BI__builtin_classify_type:
return interp__builtin_classify_type(S, OpPC, Frame, Call);
case Builtin::BI__builtin_expect:
case Builtin::BI__builtin_expect_with_probability:
return interp__builtin_expect(S, OpPC, Frame, Call);
case Builtin::BI__builtin_rotateleft8:
case Builtin::BI__builtin_rotateleft16:
case Builtin::BI__builtin_rotateleft32:
case Builtin::BI__builtin_rotateleft64:
case Builtin::BI_rotl8: // Microsoft variants of rotate left
case Builtin::BI_rotl16:
case Builtin::BI_rotl:
case Builtin::BI_lrotl:
case Builtin::BI_rotl64:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &Value, const APSInt &Amount) {
return Value.rotl(Amount);
});
case Builtin::BI__builtin_rotateright8:
case Builtin::BI__builtin_rotateright16:
case Builtin::BI__builtin_rotateright32:
case Builtin::BI__builtin_rotateright64:
case Builtin::BI_rotr8: // Microsoft variants of rotate right
case Builtin::BI_rotr16:
case Builtin::BI_rotr:
case Builtin::BI_lrotr:
case Builtin::BI_rotr64:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &Value, const APSInt &Amount) {
return Value.rotr(Amount);
});
case Builtin::BI__builtin_ffs:
case Builtin::BI__builtin_ffsl:
case Builtin::BI__builtin_ffsll:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Val) {
return APInt(Val.getBitWidth(),
Val.isZero() ? 0u : Val.countTrailingZeros() + 1u);
});
case Builtin::BIaddressof:
case Builtin::BI__addressof:
case Builtin::BI__builtin_addressof:
assert(isNoopBuiltin(BuiltinID));
return interp__builtin_addressof(S, OpPC, Frame, Call);
case Builtin::BIas_const:
case Builtin::BIforward:
case Builtin::BIforward_like:
case Builtin::BImove:
case Builtin::BImove_if_noexcept:
assert(isNoopBuiltin(BuiltinID));
return interp__builtin_move(S, OpPC, Frame, Call);
case Builtin::BI__builtin_eh_return_data_regno:
return interp__builtin_eh_return_data_regno(S, OpPC, Frame, Call);
case Builtin::BI__builtin_launder:
assert(isNoopBuiltin(BuiltinID));
return true;
case Builtin::BI__builtin_add_overflow:
case Builtin::BI__builtin_sub_overflow:
case Builtin::BI__builtin_mul_overflow:
case Builtin::BI__builtin_sadd_overflow:
case Builtin::BI__builtin_uadd_overflow:
case Builtin::BI__builtin_uaddl_overflow:
case Builtin::BI__builtin_uaddll_overflow:
case Builtin::BI__builtin_usub_overflow:
case Builtin::BI__builtin_usubl_overflow:
case Builtin::BI__builtin_usubll_overflow:
case Builtin::BI__builtin_umul_overflow:
case Builtin::BI__builtin_umull_overflow:
case Builtin::BI__builtin_umulll_overflow:
case Builtin::BI__builtin_saddl_overflow:
case Builtin::BI__builtin_saddll_overflow:
case Builtin::BI__builtin_ssub_overflow:
case Builtin::BI__builtin_ssubl_overflow:
case Builtin::BI__builtin_ssubll_overflow:
case Builtin::BI__builtin_smul_overflow:
case Builtin::BI__builtin_smull_overflow:
case Builtin::BI__builtin_smulll_overflow:
return interp__builtin_overflowop(S, OpPC, Call, BuiltinID);
case Builtin::BI__builtin_addcb:
case Builtin::BI__builtin_addcs:
case Builtin::BI__builtin_addc:
case Builtin::BI__builtin_addcl:
case Builtin::BI__builtin_addcll:
case Builtin::BI__builtin_subcb:
case Builtin::BI__builtin_subcs:
case Builtin::BI__builtin_subc:
case Builtin::BI__builtin_subcl:
case Builtin::BI__builtin_subcll:
return interp__builtin_carryop(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__builtin_clz:
case Builtin::BI__builtin_clzl:
case Builtin::BI__builtin_clzll:
case Builtin::BI__builtin_clzs:
case Builtin::BI__builtin_clzg:
case Builtin::BI__lzcnt16: // Microsoft variants of count leading-zeroes
case Builtin::BI__lzcnt:
case Builtin::BI__lzcnt64:
return interp__builtin_clz(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__builtin_ctz:
case Builtin::BI__builtin_ctzl:
case Builtin::BI__builtin_ctzll:
case Builtin::BI__builtin_ctzs:
case Builtin::BI__builtin_ctzg:
return interp__builtin_ctz(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__builtin_elementwise_clzg:
case Builtin::BI__builtin_elementwise_ctzg:
return interp__builtin_elementwise_countzeroes(S, OpPC, Frame, Call,
BuiltinID);
case Builtin::BI__builtin_bswapg:
case Builtin::BI__builtin_bswap16:
case Builtin::BI__builtin_bswap32:
case Builtin::BI__builtin_bswap64:
return interp__builtin_bswap(S, OpPC, Frame, Call);
case Builtin::BI__atomic_always_lock_free:
case Builtin::BI__atomic_is_lock_free:
return interp__builtin_atomic_lock_free(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__c11_atomic_is_lock_free:
return interp__builtin_c11_atomic_is_lock_free(S, OpPC, Frame, Call);
case Builtin::BI__builtin_complex:
return interp__builtin_complex(S, OpPC, Frame, Call);
case Builtin::BI__builtin_is_aligned:
case Builtin::BI__builtin_align_up:
case Builtin::BI__builtin_align_down:
return interp__builtin_is_aligned_up_down(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__builtin_assume_aligned:
return interp__builtin_assume_aligned(S, OpPC, Frame, Call);
case clang::X86::BI__builtin_ia32_bextr_u32:
case clang::X86::BI__builtin_ia32_bextr_u64:
case clang::X86::BI__builtin_ia32_bextri_u32:
case clang::X86::BI__builtin_ia32_bextri_u64:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &Val, const APSInt &Idx) {
unsigned BitWidth = Val.getBitWidth();
uint64_t Shift = Idx.extractBitsAsZExtValue(8, 0);
uint64_t Length = Idx.extractBitsAsZExtValue(8, 8);
if (Length > BitWidth) {
Length = BitWidth;
}
// Handle out of bounds cases.
if (Length == 0 || Shift >= BitWidth)
return APInt(BitWidth, 0);
uint64_t Result = Val.getZExtValue() >> Shift;
Result &= llvm::maskTrailingOnes<uint64_t>(Length);
return APInt(BitWidth, Result);
});
case clang::X86::BI__builtin_ia32_bzhi_si:
case clang::X86::BI__builtin_ia32_bzhi_di:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &Val, const APSInt &Idx) {
unsigned BitWidth = Val.getBitWidth();
uint64_t Index = Idx.extractBitsAsZExtValue(8, 0);
APSInt Result = Val;
if (Index < BitWidth)
Result.clearHighBits(BitWidth - Index);
return Result;
});
case clang::X86::BI__builtin_ia32_ktestcqi:
case clang::X86::BI__builtin_ia32_ktestchi:
case clang::X86::BI__builtin_ia32_ktestcsi:
case clang::X86::BI__builtin_ia32_ktestcdi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
return APInt(sizeof(unsigned char) * 8, (~A & B) == 0);
});
case clang::X86::BI__builtin_ia32_ktestzqi:
case clang::X86::BI__builtin_ia32_ktestzhi:
case clang::X86::BI__builtin_ia32_ktestzsi:
case clang::X86::BI__builtin_ia32_ktestzdi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
return APInt(sizeof(unsigned char) * 8, (A & B) == 0);
});
case clang::X86::BI__builtin_ia32_kortestcqi:
case clang::X86::BI__builtin_ia32_kortestchi:
case clang::X86::BI__builtin_ia32_kortestcsi:
case clang::X86::BI__builtin_ia32_kortestcdi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
return APInt(sizeof(unsigned char) * 8, ~(A | B) == 0);
});
case clang::X86::BI__builtin_ia32_kortestzqi:
case clang::X86::BI__builtin_ia32_kortestzhi:
case clang::X86::BI__builtin_ia32_kortestzsi:
case clang::X86::BI__builtin_ia32_kortestzdi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
return APInt(sizeof(unsigned char) * 8, (A | B) == 0);
});
case clang::X86::BI__builtin_ia32_lzcnt_u16:
case clang::X86::BI__builtin_ia32_lzcnt_u32:
case clang::X86::BI__builtin_ia32_lzcnt_u64:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Src) {
return APInt(Src.getBitWidth(), Src.countLeadingZeros());
});
case clang::X86::BI__builtin_ia32_tzcnt_u16:
case clang::X86::BI__builtin_ia32_tzcnt_u32:
case clang::X86::BI__builtin_ia32_tzcnt_u64:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Src) {
return APInt(Src.getBitWidth(), Src.countTrailingZeros());
});
case clang::X86::BI__builtin_ia32_pdep_si:
case clang::X86::BI__builtin_ia32_pdep_di:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &Val, const APSInt &Mask) {
unsigned BitWidth = Val.getBitWidth();
APInt Result = APInt::getZero(BitWidth);
for (unsigned I = 0, P = 0; I != BitWidth; ++I) {
if (Mask[I])
Result.setBitVal(I, Val[P++]);
}
return Result;
});
case clang::X86::BI__builtin_ia32_pext_si:
case clang::X86::BI__builtin_ia32_pext_di:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &Val, const APSInt &Mask) {
unsigned BitWidth = Val.getBitWidth();
APInt Result = APInt::getZero(BitWidth);
for (unsigned I = 0, P = 0; I != BitWidth; ++I) {
if (Mask[I])
Result.setBitVal(P++, Val[I]);
}
return Result;
});
case clang::X86::BI__builtin_ia32_addcarryx_u32:
case clang::X86::BI__builtin_ia32_addcarryx_u64:
case clang::X86::BI__builtin_ia32_subborrow_u32:
case clang::X86::BI__builtin_ia32_subborrow_u64:
return interp__builtin_ia32_addcarry_subborrow(S, OpPC, Frame, Call,
BuiltinID);
case Builtin::BI__builtin_os_log_format_buffer_size:
return interp__builtin_os_log_format_buffer_size(S, OpPC, Frame, Call);
case Builtin::BI__builtin_ptrauth_string_discriminator:
return interp__builtin_ptrauth_string_discriminator(S, OpPC, Frame, Call);
case Builtin::BI__builtin_infer_alloc_token:
return interp__builtin_infer_alloc_token(S, OpPC, Frame, Call);
case Builtin::BI__noop:
pushInteger(S, 0, Call->getType());
return true;
case Builtin::BI__builtin_operator_new:
return interp__builtin_operator_new(S, OpPC, Frame, Call);
case Builtin::BI__builtin_operator_delete:
return interp__builtin_operator_delete(S, OpPC, Frame, Call);
case Builtin::BI__arithmetic_fence:
return interp__builtin_arithmetic_fence(S, OpPC, Frame, Call);
case Builtin::BI__builtin_reduce_add:
case Builtin::BI__builtin_reduce_mul:
case Builtin::BI__builtin_reduce_and:
case Builtin::BI__builtin_reduce_or:
case Builtin::BI__builtin_reduce_xor:
case Builtin::BI__builtin_reduce_min:
case Builtin::BI__builtin_reduce_max:
return interp__builtin_vector_reduce(S, OpPC, Call, BuiltinID);
case Builtin::BI__builtin_elementwise_popcount:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Src) {
return APInt(Src.getBitWidth(), Src.popcount());
});
case Builtin::BI__builtin_elementwise_bitreverse:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Src) { return Src.reverseBits(); });
case Builtin::BI__builtin_elementwise_abs:
return interp__builtin_elementwise_abs(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__builtin_memcpy:
case Builtin::BImemcpy:
case Builtin::BI__builtin_wmemcpy:
case Builtin::BIwmemcpy:
case Builtin::BI__builtin_memmove:
case Builtin::BImemmove:
case Builtin::BI__builtin_wmemmove:
case Builtin::BIwmemmove:
return interp__builtin_memcpy(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BI__builtin_memcmp:
case Builtin::BImemcmp:
case Builtin::BI__builtin_bcmp:
case Builtin::BIbcmp:
case Builtin::BI__builtin_wmemcmp:
case Builtin::BIwmemcmp:
return interp__builtin_memcmp(S, OpPC, Frame, Call, BuiltinID);
case Builtin::BImemchr:
case Builtin::BI__builtin_memchr:
case Builtin::BIstrchr:
case Builtin::BI__builtin_strchr:
case Builtin::BIwmemchr:
case Builtin::BI__builtin_wmemchr:
case Builtin::BIwcschr:
case Builtin::BI__builtin_wcschr:
case Builtin::BI__builtin_char_memchr:
return interp__builtin_memchr(S, OpPC, Call, BuiltinID);
case Builtin::BI__builtin_object_size:
case Builtin::BI__builtin_dynamic_object_size:
return interp__builtin_object_size(S, OpPC, Frame, Call);
case Builtin::BI__builtin_is_within_lifetime:
return interp__builtin_is_within_lifetime(S, OpPC, Call);
case Builtin::BI__builtin_elementwise_add_sat:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
return LHS.isSigned() ? LHS.sadd_sat(RHS) : LHS.uadd_sat(RHS);
});
case Builtin::BI__builtin_elementwise_sub_sat:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
return LHS.isSigned() ? LHS.ssub_sat(RHS) : LHS.usub_sat(RHS);
});
case X86::BI__builtin_ia32_extract128i256:
case X86::BI__builtin_ia32_vextractf128_pd256:
case X86::BI__builtin_ia32_vextractf128_ps256:
case X86::BI__builtin_ia32_vextractf128_si256:
return interp__builtin_x86_extract_vector(S, OpPC, Call, BuiltinID);
case X86::BI__builtin_ia32_extractf32x4_256_mask:
case X86::BI__builtin_ia32_extractf32x4_mask:
case X86::BI__builtin_ia32_extractf32x8_mask:
case X86::BI__builtin_ia32_extractf64x2_256_mask:
case X86::BI__builtin_ia32_extractf64x2_512_mask:
case X86::BI__builtin_ia32_extractf64x4_mask:
case X86::BI__builtin_ia32_extracti32x4_256_mask:
case X86::BI__builtin_ia32_extracti32x4_mask:
case X86::BI__builtin_ia32_extracti32x8_mask:
case X86::BI__builtin_ia32_extracti64x2_256_mask:
case X86::BI__builtin_ia32_extracti64x2_512_mask:
case X86::BI__builtin_ia32_extracti64x4_mask:
return interp__builtin_x86_extract_vector_masked(S, OpPC, Call, BuiltinID);
case clang::X86::BI__builtin_ia32_pmulhrsw128:
case clang::X86::BI__builtin_ia32_pmulhrsw256:
case clang::X86::BI__builtin_ia32_pmulhrsw512:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
return (llvm::APIntOps::mulsExtended(LHS, RHS).ashr(14) + 1)
.extractBits(16, 1);
});
case clang::X86::BI__builtin_ia32_movmskps:
case clang::X86::BI__builtin_ia32_movmskpd:
case clang::X86::BI__builtin_ia32_pmovmskb128:
case clang::X86::BI__builtin_ia32_pmovmskb256:
case clang::X86::BI__builtin_ia32_movmskps256:
case clang::X86::BI__builtin_ia32_movmskpd256: {
return interp__builtin_ia32_movmsk_op(S, OpPC, Call);
}
case X86::BI__builtin_ia32_psignb128:
case X86::BI__builtin_ia32_psignb256:
case X86::BI__builtin_ia32_psignw128:
case X86::BI__builtin_ia32_psignw256:
case X86::BI__builtin_ia32_psignd128:
case X86::BI__builtin_ia32_psignd256:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APInt &AElem, const APInt &BElem) {
if (BElem.isZero())
return APInt::getZero(AElem.getBitWidth());
if (BElem.isNegative())
return -AElem;
return AElem;
});
case clang::X86::BI__builtin_ia32_pavgb128:
case clang::X86::BI__builtin_ia32_pavgw128:
case clang::X86::BI__builtin_ia32_pavgb256:
case clang::X86::BI__builtin_ia32_pavgw256:
case clang::X86::BI__builtin_ia32_pavgb512:
case clang::X86::BI__builtin_ia32_pavgw512:
return interp__builtin_elementwise_int_binop(S, OpPC, Call,
llvm::APIntOps::avgCeilU);
case clang::X86::BI__builtin_ia32_pmaddubsw128:
case clang::X86::BI__builtin_ia32_pmaddubsw256:
case clang::X86::BI__builtin_ia32_pmaddubsw512:
return interp__builtin_ia32_pmul(
S, OpPC, Call,
[](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
const APSInt &HiRHS) {
unsigned BitWidth = 2 * LoLHS.getBitWidth();
return (LoLHS.zext(BitWidth) * LoRHS.sext(BitWidth))
.sadd_sat((HiLHS.zext(BitWidth) * HiRHS.sext(BitWidth)));
});
case clang::X86::BI__builtin_ia32_pmaddwd128:
case clang::X86::BI__builtin_ia32_pmaddwd256:
case clang::X86::BI__builtin_ia32_pmaddwd512:
return interp__builtin_ia32_pmul(
S, OpPC, Call,
[](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
const APSInt &HiRHS) {
unsigned BitWidth = 2 * LoLHS.getBitWidth();
return (LoLHS.sext(BitWidth) * LoRHS.sext(BitWidth)) +
(HiLHS.sext(BitWidth) * HiRHS.sext(BitWidth));
});
case clang::X86::BI__builtin_ia32_pmulhuw128:
case clang::X86::BI__builtin_ia32_pmulhuw256:
case clang::X86::BI__builtin_ia32_pmulhuw512:
return interp__builtin_elementwise_int_binop(S, OpPC, Call,
llvm::APIntOps::mulhu);
case clang::X86::BI__builtin_ia32_pmulhw128:
case clang::X86::BI__builtin_ia32_pmulhw256:
case clang::X86::BI__builtin_ia32_pmulhw512:
return interp__builtin_elementwise_int_binop(S, OpPC, Call,
llvm::APIntOps::mulhs);
case clang::X86::BI__builtin_ia32_psllv2di:
case clang::X86::BI__builtin_ia32_psllv4di:
case clang::X86::BI__builtin_ia32_psllv4si:
case clang::X86::BI__builtin_ia32_psllv8di:
case clang::X86::BI__builtin_ia32_psllv8hi:
case clang::X86::BI__builtin_ia32_psllv8si:
case clang::X86::BI__builtin_ia32_psllv16hi:
case clang::X86::BI__builtin_ia32_psllv16si:
case clang::X86::BI__builtin_ia32_psllv32hi:
case clang::X86::BI__builtin_ia32_psllwi128:
case clang::X86::BI__builtin_ia32_psllwi256:
case clang::X86::BI__builtin_ia32_psllwi512:
case clang::X86::BI__builtin_ia32_pslldi128:
case clang::X86::BI__builtin_ia32_pslldi256:
case clang::X86::BI__builtin_ia32_pslldi512:
case clang::X86::BI__builtin_ia32_psllqi128:
case clang::X86::BI__builtin_ia32_psllqi256:
case clang::X86::BI__builtin_ia32_psllqi512:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
if (RHS.uge(LHS.getBitWidth())) {
return APInt::getZero(LHS.getBitWidth());
}
return LHS.shl(RHS.getZExtValue());
});
case clang::X86::BI__builtin_ia32_psrav4si:
case clang::X86::BI__builtin_ia32_psrav8di:
case clang::X86::BI__builtin_ia32_psrav8hi:
case clang::X86::BI__builtin_ia32_psrav8si:
case clang::X86::BI__builtin_ia32_psrav16hi:
case clang::X86::BI__builtin_ia32_psrav16si:
case clang::X86::BI__builtin_ia32_psrav32hi:
case clang::X86::BI__builtin_ia32_psravq128:
case clang::X86::BI__builtin_ia32_psravq256:
case clang::X86::BI__builtin_ia32_psrawi128:
case clang::X86::BI__builtin_ia32_psrawi256:
case clang::X86::BI__builtin_ia32_psrawi512:
case clang::X86::BI__builtin_ia32_psradi128:
case clang::X86::BI__builtin_ia32_psradi256:
case clang::X86::BI__builtin_ia32_psradi512:
case clang::X86::BI__builtin_ia32_psraqi128:
case clang::X86::BI__builtin_ia32_psraqi256:
case clang::X86::BI__builtin_ia32_psraqi512:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
if (RHS.uge(LHS.getBitWidth())) {
return LHS.ashr(LHS.getBitWidth() - 1);
}
return LHS.ashr(RHS.getZExtValue());
});
case clang::X86::BI__builtin_ia32_psrlv2di:
case clang::X86::BI__builtin_ia32_psrlv4di:
case clang::X86::BI__builtin_ia32_psrlv4si:
case clang::X86::BI__builtin_ia32_psrlv8di:
case clang::X86::BI__builtin_ia32_psrlv8hi:
case clang::X86::BI__builtin_ia32_psrlv8si:
case clang::X86::BI__builtin_ia32_psrlv16hi:
case clang::X86::BI__builtin_ia32_psrlv16si:
case clang::X86::BI__builtin_ia32_psrlv32hi:
case clang::X86::BI__builtin_ia32_psrlwi128:
case clang::X86::BI__builtin_ia32_psrlwi256:
case clang::X86::BI__builtin_ia32_psrlwi512:
case clang::X86::BI__builtin_ia32_psrldi128:
case clang::X86::BI__builtin_ia32_psrldi256:
case clang::X86::BI__builtin_ia32_psrldi512:
case clang::X86::BI__builtin_ia32_psrlqi128:
case clang::X86::BI__builtin_ia32_psrlqi256:
case clang::X86::BI__builtin_ia32_psrlqi512:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &LHS, const APSInt &RHS) {
if (RHS.uge(LHS.getBitWidth())) {
return APInt::getZero(LHS.getBitWidth());
}
return LHS.lshr(RHS.getZExtValue());
});
case clang::X86::BI__builtin_ia32_packsswb128:
case clang::X86::BI__builtin_ia32_packsswb256:
case clang::X86::BI__builtin_ia32_packsswb512:
case clang::X86::BI__builtin_ia32_packssdw128:
case clang::X86::BI__builtin_ia32_packssdw256:
case clang::X86::BI__builtin_ia32_packssdw512:
return interp__builtin_x86_pack(S, OpPC, Call, [](const APSInt &Src) {
return APInt(Src).truncSSat(Src.getBitWidth() / 2);
});
case clang::X86::BI__builtin_ia32_packusdw128:
case clang::X86::BI__builtin_ia32_packusdw256:
case clang::X86::BI__builtin_ia32_packusdw512:
case clang::X86::BI__builtin_ia32_packuswb128:
case clang::X86::BI__builtin_ia32_packuswb256:
case clang::X86::BI__builtin_ia32_packuswb512:
return interp__builtin_x86_pack(S, OpPC, Call, [](const APSInt &Src) {
unsigned DstBits = Src.getBitWidth() / 2;
if (Src.isNegative())
return APInt::getZero(DstBits);
if (Src.isIntN(DstBits))
return APInt(Src).trunc(DstBits);
return APInt::getAllOnes(DstBits);
});
case clang::X86::BI__builtin_ia32_selectss_128:
case clang::X86::BI__builtin_ia32_selectsd_128:
case clang::X86::BI__builtin_ia32_selectsh_128:
case clang::X86::BI__builtin_ia32_selectsbf_128:
return interp__builtin_select_scalar(S, Call);
case clang::X86::BI__builtin_ia32_vprotbi:
case clang::X86::BI__builtin_ia32_vprotdi:
case clang::X86::BI__builtin_ia32_vprotqi:
case clang::X86::BI__builtin_ia32_vprotwi:
case clang::X86::BI__builtin_ia32_prold128:
case clang::X86::BI__builtin_ia32_prold256:
case clang::X86::BI__builtin_ia32_prold512:
case clang::X86::BI__builtin_ia32_prolq128:
case clang::X86::BI__builtin_ia32_prolq256:
case clang::X86::BI__builtin_ia32_prolq512:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS.rotl(RHS); });
case clang::X86::BI__builtin_ia32_prord128:
case clang::X86::BI__builtin_ia32_prord256:
case clang::X86::BI__builtin_ia32_prord512:
case clang::X86::BI__builtin_ia32_prorq128:
case clang::X86::BI__builtin_ia32_prorq256:
case clang::X86::BI__builtin_ia32_prorq512:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS.rotr(RHS); });
case Builtin::BI__builtin_elementwise_max:
case Builtin::BI__builtin_elementwise_min:
return interp__builtin_elementwise_maxmin(S, OpPC, Call, BuiltinID);
case clang::X86::BI__builtin_ia32_phaddw128:
case clang::X86::BI__builtin_ia32_phaddw256:
case clang::X86::BI__builtin_ia32_phaddd128:
case clang::X86::BI__builtin_ia32_phaddd256:
return interp_builtin_horizontal_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
case clang::X86::BI__builtin_ia32_phaddsw128:
case clang::X86::BI__builtin_ia32_phaddsw256:
return interp_builtin_horizontal_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS.sadd_sat(RHS); });
case clang::X86::BI__builtin_ia32_phsubw128:
case clang::X86::BI__builtin_ia32_phsubw256:
case clang::X86::BI__builtin_ia32_phsubd128:
case clang::X86::BI__builtin_ia32_phsubd256:
return interp_builtin_horizontal_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS - RHS; });
case clang::X86::BI__builtin_ia32_phsubsw128:
case clang::X86::BI__builtin_ia32_phsubsw256:
return interp_builtin_horizontal_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS.ssub_sat(RHS); });
case clang::X86::BI__builtin_ia32_haddpd:
case clang::X86::BI__builtin_ia32_haddps:
case clang::X86::BI__builtin_ia32_haddpd256:
case clang::X86::BI__builtin_ia32_haddps256:
return interp_builtin_horizontal_fp_binop(
S, OpPC, Call,
[](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
APFloat F = LHS;
F.add(RHS, RM);
return F;
});
case clang::X86::BI__builtin_ia32_hsubpd:
case clang::X86::BI__builtin_ia32_hsubps:
case clang::X86::BI__builtin_ia32_hsubpd256:
case clang::X86::BI__builtin_ia32_hsubps256:
return interp_builtin_horizontal_fp_binop(
S, OpPC, Call,
[](const APFloat &LHS, const APFloat &RHS, llvm::RoundingMode RM) {
APFloat F = LHS;
F.subtract(RHS, RM);
return F;
});
case clang::X86::BI__builtin_ia32_addsubpd:
case clang::X86::BI__builtin_ia32_addsubps:
case clang::X86::BI__builtin_ia32_addsubpd256:
case clang::X86::BI__builtin_ia32_addsubps256:
return interp__builtin_ia32_addsub(S, OpPC, Call);
case clang::X86::BI__builtin_ia32_pmuldq128:
case clang::X86::BI__builtin_ia32_pmuldq256:
case clang::X86::BI__builtin_ia32_pmuldq512:
return interp__builtin_ia32_pmul(
S, OpPC, Call,
[](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
const APSInt &HiRHS) {
return llvm::APIntOps::mulsExtended(LoLHS, LoRHS);
});
case clang::X86::BI__builtin_ia32_pmuludq128:
case clang::X86::BI__builtin_ia32_pmuludq256:
case clang::X86::BI__builtin_ia32_pmuludq512:
return interp__builtin_ia32_pmul(
S, OpPC, Call,
[](const APSInt &LoLHS, const APSInt &HiLHS, const APSInt &LoRHS,
const APSInt &HiRHS) {
return llvm::APIntOps::muluExtended(LoLHS, LoRHS);
});
case Builtin::BI__builtin_elementwise_fma:
return interp__builtin_elementwise_triop_fp(
S, OpPC, Call,
[](const APFloat &X, const APFloat &Y, const APFloat &Z,
llvm::RoundingMode RM) {
APFloat F = X;
F.fusedMultiplyAdd(Y, Z, RM);
return F;
});
case X86::BI__builtin_ia32_vpmadd52luq128:
case X86::BI__builtin_ia32_vpmadd52luq256:
case X86::BI__builtin_ia32_vpmadd52luq512:
return interp__builtin_elementwise_triop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {
return A + (B.trunc(52) * C.trunc(52)).zext(64);
});
case X86::BI__builtin_ia32_vpmadd52huq128:
case X86::BI__builtin_ia32_vpmadd52huq256:
case X86::BI__builtin_ia32_vpmadd52huq512:
return interp__builtin_elementwise_triop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B, const APSInt &C) {
return A + llvm::APIntOps::mulhu(B.trunc(52), C.trunc(52)).zext(64);
});
case X86::BI__builtin_ia32_vpshldd128:
case X86::BI__builtin_ia32_vpshldd256:
case X86::BI__builtin_ia32_vpshldd512:
case X86::BI__builtin_ia32_vpshldq128:
case X86::BI__builtin_ia32_vpshldq256:
case X86::BI__builtin_ia32_vpshldq512:
case X86::BI__builtin_ia32_vpshldw128:
case X86::BI__builtin_ia32_vpshldw256:
case X86::BI__builtin_ia32_vpshldw512:
return interp__builtin_elementwise_triop(
S, OpPC, Call,
[](const APSInt &Hi, const APSInt &Lo, const APSInt &Amt) {
return llvm::APIntOps::fshl(Hi, Lo, Amt);
});
case X86::BI__builtin_ia32_vpshrdd128:
case X86::BI__builtin_ia32_vpshrdd256:
case X86::BI__builtin_ia32_vpshrdd512:
case X86::BI__builtin_ia32_vpshrdq128:
case X86::BI__builtin_ia32_vpshrdq256:
case X86::BI__builtin_ia32_vpshrdq512:
case X86::BI__builtin_ia32_vpshrdw128:
case X86::BI__builtin_ia32_vpshrdw256:
case X86::BI__builtin_ia32_vpshrdw512:
// NOTE: Reversed Hi/Lo operands.
return interp__builtin_elementwise_triop(
S, OpPC, Call,
[](const APSInt &Lo, const APSInt &Hi, const APSInt &Amt) {
return llvm::APIntOps::fshr(Hi, Lo, Amt);
});
case X86::BI__builtin_ia32_vpconflictsi_128:
case X86::BI__builtin_ia32_vpconflictsi_256:
case X86::BI__builtin_ia32_vpconflictsi_512:
case X86::BI__builtin_ia32_vpconflictdi_128:
case X86::BI__builtin_ia32_vpconflictdi_256:
case X86::BI__builtin_ia32_vpconflictdi_512:
return interp__builtin_ia32_vpconflict(S, OpPC, Call);
case clang::X86::BI__builtin_ia32_blendpd:
case clang::X86::BI__builtin_ia32_blendpd256:
case clang::X86::BI__builtin_ia32_blendps:
case clang::X86::BI__builtin_ia32_blendps256:
case clang::X86::BI__builtin_ia32_pblendw128:
case clang::X86::BI__builtin_ia32_pblendw256:
case clang::X86::BI__builtin_ia32_pblendd128:
case clang::X86::BI__builtin_ia32_pblendd256:
return interp__builtin_blend(S, OpPC, Call);
case clang::X86::BI__builtin_ia32_blendvpd:
case clang::X86::BI__builtin_ia32_blendvpd256:
case clang::X86::BI__builtin_ia32_blendvps:
case clang::X86::BI__builtin_ia32_blendvps256:
return interp__builtin_elementwise_triop_fp(
S, OpPC, Call,
[](const APFloat &F, const APFloat &T, const APFloat &C,
llvm::RoundingMode) { return C.isNegative() ? T : F; });
case clang::X86::BI__builtin_ia32_pblendvb128:
case clang::X86::BI__builtin_ia32_pblendvb256:
return interp__builtin_elementwise_triop(
S, OpPC, Call, [](const APSInt &F, const APSInt &T, const APSInt &C) {
return ((APInt)C).isNegative() ? T : F;
});
case X86::BI__builtin_ia32_ptestz128:
case X86::BI__builtin_ia32_ptestz256:
case X86::BI__builtin_ia32_vtestzps:
case X86::BI__builtin_ia32_vtestzps256:
case X86::BI__builtin_ia32_vtestzpd:
case X86::BI__builtin_ia32_vtestzpd256:
return interp__builtin_ia32_test_op(
S, OpPC, Call,
[](const APInt &A, const APInt &B) { return (A & B) == 0; });
case X86::BI__builtin_ia32_ptestc128:
case X86::BI__builtin_ia32_ptestc256:
case X86::BI__builtin_ia32_vtestcps:
case X86::BI__builtin_ia32_vtestcps256:
case X86::BI__builtin_ia32_vtestcpd:
case X86::BI__builtin_ia32_vtestcpd256:
return interp__builtin_ia32_test_op(
S, OpPC, Call,
[](const APInt &A, const APInt &B) { return (~A & B) == 0; });
case X86::BI__builtin_ia32_ptestnzc128:
case X86::BI__builtin_ia32_ptestnzc256:
case X86::BI__builtin_ia32_vtestnzcps:
case X86::BI__builtin_ia32_vtestnzcps256:
case X86::BI__builtin_ia32_vtestnzcpd:
case X86::BI__builtin_ia32_vtestnzcpd256:
return interp__builtin_ia32_test_op(
S, OpPC, Call, [](const APInt &A, const APInt &B) {
return ((A & B) != 0) && ((~A & B) != 0);
});
case X86::BI__builtin_ia32_selectb_128:
case X86::BI__builtin_ia32_selectb_256:
case X86::BI__builtin_ia32_selectb_512:
case X86::BI__builtin_ia32_selectw_128:
case X86::BI__builtin_ia32_selectw_256:
case X86::BI__builtin_ia32_selectw_512:
case X86::BI__builtin_ia32_selectd_128:
case X86::BI__builtin_ia32_selectd_256:
case X86::BI__builtin_ia32_selectd_512:
case X86::BI__builtin_ia32_selectq_128:
case X86::BI__builtin_ia32_selectq_256:
case X86::BI__builtin_ia32_selectq_512:
case X86::BI__builtin_ia32_selectph_128:
case X86::BI__builtin_ia32_selectph_256:
case X86::BI__builtin_ia32_selectph_512:
case X86::BI__builtin_ia32_selectpbf_128:
case X86::BI__builtin_ia32_selectpbf_256:
case X86::BI__builtin_ia32_selectpbf_512:
case X86::BI__builtin_ia32_selectps_128:
case X86::BI__builtin_ia32_selectps_256:
case X86::BI__builtin_ia32_selectps_512:
case X86::BI__builtin_ia32_selectpd_128:
case X86::BI__builtin_ia32_selectpd_256:
case X86::BI__builtin_ia32_selectpd_512:
return interp__builtin_select(S, OpPC, Call);
case X86::BI__builtin_ia32_shufps:
case X86::BI__builtin_ia32_shufps256:
case X86::BI__builtin_ia32_shufps512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
unsigned NumElemPerLane = 4;
unsigned NumSelectableElems = NumElemPerLane / 2;
unsigned BitsPerElem = 2;
unsigned IndexMask = 0x3;
unsigned MaskBits = 8;
unsigned Lane = DstIdx / NumElemPerLane;
unsigned ElemInLane = DstIdx % NumElemPerLane;
unsigned LaneOffset = Lane * NumElemPerLane;
unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
return std::pair<unsigned, int>{SrcIdx,
static_cast<int>(LaneOffset + Index)};
});
case X86::BI__builtin_ia32_shufpd:
case X86::BI__builtin_ia32_shufpd256:
case X86::BI__builtin_ia32_shufpd512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
unsigned NumElemPerLane = 2;
unsigned NumSelectableElems = NumElemPerLane / 2;
unsigned BitsPerElem = 1;
unsigned IndexMask = 0x1;
unsigned MaskBits = 8;
unsigned Lane = DstIdx / NumElemPerLane;
unsigned ElemInLane = DstIdx % NumElemPerLane;
unsigned LaneOffset = Lane * NumElemPerLane;
unsigned SrcIdx = ElemInLane >= NumSelectableElems ? 1 : 0;
unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
unsigned Index = (ShuffleMask >> BitIndex) & IndexMask;
return std::pair<unsigned, int>{SrcIdx,
static_cast<int>(LaneOffset + Index)};
});
case X86::BI__builtin_ia32_insertps128:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned Mask) {
// Bits [3:0]: zero mask - if bit is set, zero this element
if ((Mask & (1 << DstIdx)) != 0) {
return std::pair<unsigned, int>{0, -1};
}
// Bits [7:6]: select element from source vector Y (0-3)
// Bits [5:4]: select destination position (0-3)
unsigned SrcElem = (Mask >> 6) & 0x3;
unsigned DstElem = (Mask >> 4) & 0x3;
if (DstIdx == DstElem) {
// Insert element from source vector (B) at this position
return std::pair<unsigned, int>{1, static_cast<int>(SrcElem)};
} else {
// Copy from destination vector (A)
return std::pair<unsigned, int>{0, static_cast<int>(DstIdx)};
}
});
case X86::BI__builtin_ia32_permvarsi256:
case X86::BI__builtin_ia32_permvarsf256:
case X86::BI__builtin_ia32_permvardf512:
case X86::BI__builtin_ia32_permvardi512:
case X86::BI__builtin_ia32_permvarhi128:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x7;
return std::pair<unsigned, int>{0, Offset};
});
case X86::BI__builtin_ia32_permvarqi128:
case X86::BI__builtin_ia32_permvarhi256:
case X86::BI__builtin_ia32_permvarsi512:
case X86::BI__builtin_ia32_permvarsf512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0xF;
return std::pair<unsigned, int>{0, Offset};
});
case X86::BI__builtin_ia32_permvardi256:
case X86::BI__builtin_ia32_permvardf256:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x3;
return std::pair<unsigned, int>{0, Offset};
});
case X86::BI__builtin_ia32_permvarqi256:
case X86::BI__builtin_ia32_permvarhi512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x1F;
return std::pair<unsigned, int>{0, Offset};
});
case X86::BI__builtin_ia32_permvarqi512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x3F;
return std::pair<unsigned, int>{0, Offset};
});
case X86::BI__builtin_ia32_vpermi2varq128:
case X86::BI__builtin_ia32_vpermi2varpd128:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x1;
unsigned SrcIdx = (ShuffleMask >> 1) & 0x1;
return std::pair<unsigned, int>{SrcIdx, Offset};
});
case X86::BI__builtin_ia32_vpermi2vard128:
case X86::BI__builtin_ia32_vpermi2varps128:
case X86::BI__builtin_ia32_vpermi2varq256:
case X86::BI__builtin_ia32_vpermi2varpd256:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x3;
unsigned SrcIdx = (ShuffleMask >> 2) & 0x1;
return std::pair<unsigned, int>{SrcIdx, Offset};
});
case X86::BI__builtin_ia32_vpermi2varhi128:
case X86::BI__builtin_ia32_vpermi2vard256:
case X86::BI__builtin_ia32_vpermi2varps256:
case X86::BI__builtin_ia32_vpermi2varq512:
case X86::BI__builtin_ia32_vpermi2varpd512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x7;
unsigned SrcIdx = (ShuffleMask >> 3) & 0x1;
return std::pair<unsigned, int>{SrcIdx, Offset};
});
case X86::BI__builtin_ia32_vpermi2varqi128:
case X86::BI__builtin_ia32_vpermi2varhi256:
case X86::BI__builtin_ia32_vpermi2vard512:
case X86::BI__builtin_ia32_vpermi2varps512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0xF;
unsigned SrcIdx = (ShuffleMask >> 4) & 0x1;
return std::pair<unsigned, int>{SrcIdx, Offset};
});
case X86::BI__builtin_ia32_vpermi2varqi256:
case X86::BI__builtin_ia32_vpermi2varhi512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x1F;
unsigned SrcIdx = (ShuffleMask >> 5) & 0x1;
return std::pair<unsigned, int>{SrcIdx, Offset};
});
case X86::BI__builtin_ia32_vpermi2varqi512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
int Offset = ShuffleMask & 0x3F;
unsigned SrcIdx = (ShuffleMask >> 6) & 0x1;
return std::pair<unsigned, int>{SrcIdx, Offset};
});
case X86::BI__builtin_ia32_pshufb128:
case X86::BI__builtin_ia32_pshufb256:
case X86::BI__builtin_ia32_pshufb512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
uint8_t Ctlb = static_cast<uint8_t>(ShuffleMask);
if (Ctlb & 0x80)
return std::make_pair(0, -1);
unsigned LaneBase = (DstIdx / 16) * 16;
unsigned SrcOffset = Ctlb & 0x0F;
unsigned SrcIdx = LaneBase + SrcOffset;
return std::make_pair(0, static_cast<int>(SrcIdx));
});
case X86::BI__builtin_ia32_pshuflw:
case X86::BI__builtin_ia32_pshuflw256:
case X86::BI__builtin_ia32_pshuflw512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
unsigned LaneBase = (DstIdx / 8) * 8;
unsigned LaneIdx = DstIdx % 8;
if (LaneIdx < 4) {
unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
return std::make_pair(0, static_cast<int>(LaneBase + Sel));
}
return std::make_pair(0, static_cast<int>(DstIdx));
});
case X86::BI__builtin_ia32_pshufhw:
case X86::BI__builtin_ia32_pshufhw256:
case X86::BI__builtin_ia32_pshufhw512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
unsigned LaneBase = (DstIdx / 8) * 8;
unsigned LaneIdx = DstIdx % 8;
if (LaneIdx >= 4) {
unsigned Sel = (ShuffleMask >> (2 * (LaneIdx - 4))) & 0x3;
return std::make_pair(0, static_cast<int>(LaneBase + 4 + Sel));
}
return std::make_pair(0, static_cast<int>(DstIdx));
});
case X86::BI__builtin_ia32_pshufd:
case X86::BI__builtin_ia32_pshufd256:
case X86::BI__builtin_ia32_pshufd512:
case X86::BI__builtin_ia32_vpermilps:
case X86::BI__builtin_ia32_vpermilps256:
case X86::BI__builtin_ia32_vpermilps512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
unsigned LaneBase = (DstIdx / 4) * 4;
unsigned LaneIdx = DstIdx % 4;
unsigned Sel = (ShuffleMask >> (2 * LaneIdx)) & 0x3;
return std::make_pair(0, static_cast<int>(LaneBase + Sel));
});
case X86::BI__builtin_ia32_vpermilvarpd:
case X86::BI__builtin_ia32_vpermilvarpd256:
case X86::BI__builtin_ia32_vpermilvarpd512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
unsigned NumElemPerLane = 2;
unsigned Lane = DstIdx / NumElemPerLane;
unsigned Offset = ShuffleMask & 0b10 ? 1 : 0;
return std::make_pair(
0, static_cast<int>(Lane * NumElemPerLane + Offset));
});
case X86::BI__builtin_ia32_vpermilvarps:
case X86::BI__builtin_ia32_vpermilvarps256:
case X86::BI__builtin_ia32_vpermilvarps512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned ShuffleMask) {
unsigned NumElemPerLane = 4;
unsigned Lane = DstIdx / NumElemPerLane;
unsigned Offset = ShuffleMask & 0b11;
return std::make_pair(
0, static_cast<int>(Lane * NumElemPerLane + Offset));
});
case X86::BI__builtin_ia32_vpermilpd:
case X86::BI__builtin_ia32_vpermilpd256:
case X86::BI__builtin_ia32_vpermilpd512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned Control) {
unsigned NumElemPerLane = 2;
unsigned BitsPerElem = 1;
unsigned MaskBits = 8;
unsigned IndexMask = 0x1;
unsigned Lane = DstIdx / NumElemPerLane;
unsigned LaneOffset = Lane * NumElemPerLane;
unsigned BitIndex = (DstIdx * BitsPerElem) % MaskBits;
unsigned Index = (Control >> BitIndex) & IndexMask;
return std::make_pair(0, static_cast<int>(LaneOffset + Index));
});
case X86::BI__builtin_ia32_vpmultishiftqb128:
case X86::BI__builtin_ia32_vpmultishiftqb256:
case X86::BI__builtin_ia32_vpmultishiftqb512:
return interp__builtin_ia32_multishiftqb(S, OpPC, Call);
case X86::BI__builtin_ia32_kandqi:
case X86::BI__builtin_ia32_kandhi:
case X86::BI__builtin_ia32_kandsi:
case X86::BI__builtin_ia32_kanddi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS & RHS; });
case X86::BI__builtin_ia32_kandnqi:
case X86::BI__builtin_ia32_kandnhi:
case X86::BI__builtin_ia32_kandnsi:
case X86::BI__builtin_ia32_kandndi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return ~LHS & RHS; });
case X86::BI__builtin_ia32_korqi:
case X86::BI__builtin_ia32_korhi:
case X86::BI__builtin_ia32_korsi:
case X86::BI__builtin_ia32_kordi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS | RHS; });
case X86::BI__builtin_ia32_kxnorqi:
case X86::BI__builtin_ia32_kxnorhi:
case X86::BI__builtin_ia32_kxnorsi:
case X86::BI__builtin_ia32_kxnordi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return ~(LHS ^ RHS); });
case X86::BI__builtin_ia32_kxorqi:
case X86::BI__builtin_ia32_kxorhi:
case X86::BI__builtin_ia32_kxorsi:
case X86::BI__builtin_ia32_kxordi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS ^ RHS; });
case X86::BI__builtin_ia32_knotqi:
case X86::BI__builtin_ia32_knothi:
case X86::BI__builtin_ia32_knotsi:
case X86::BI__builtin_ia32_knotdi:
return interp__builtin_elementwise_int_unaryop(
S, OpPC, Call, [](const APSInt &Src) { return ~Src; });
case X86::BI__builtin_ia32_kaddqi:
case X86::BI__builtin_ia32_kaddhi:
case X86::BI__builtin_ia32_kaddsi:
case X86::BI__builtin_ia32_kadddi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call,
[](const APSInt &LHS, const APSInt &RHS) { return LHS + RHS; });
case X86::BI__builtin_ia32_kunpckhi:
case X86::BI__builtin_ia32_kunpckdi:
case X86::BI__builtin_ia32_kunpcksi:
return interp__builtin_elementwise_int_binop(
S, OpPC, Call, [](const APSInt &A, const APSInt &B) {
// Generic kunpack: extract lower half of each operand and concatenate
// Result = A[HalfWidth-1:0] concat B[HalfWidth-1:0]
unsigned BW = A.getBitWidth();
return APSInt(A.trunc(BW / 2).concat(B.trunc(BW / 2)),
A.isUnsigned());
});
case X86::BI__builtin_ia32_phminposuw128:
return interp__builtin_ia32_phminposuw(S, OpPC, Call);
case X86::BI__builtin_ia32_psraq128:
case X86::BI__builtin_ia32_psraq256:
case X86::BI__builtin_ia32_psraq512:
case X86::BI__builtin_ia32_psrad128:
case X86::BI__builtin_ia32_psrad256:
case X86::BI__builtin_ia32_psrad512:
case X86::BI__builtin_ia32_psraw128:
case X86::BI__builtin_ia32_psraw256:
case X86::BI__builtin_ia32_psraw512:
return interp__builtin_ia32_shift_with_count(
S, OpPC, Call,
[](const APInt &Elt, uint64_t Count) { return Elt.ashr(Count); },
[](const APInt &Elt, unsigned Width) { return Elt.ashr(Width - 1); });
case X86::BI__builtin_ia32_psllq128:
case X86::BI__builtin_ia32_psllq256:
case X86::BI__builtin_ia32_psllq512:
case X86::BI__builtin_ia32_pslld128:
case X86::BI__builtin_ia32_pslld256:
case X86::BI__builtin_ia32_pslld512:
case X86::BI__builtin_ia32_psllw128:
case X86::BI__builtin_ia32_psllw256:
case X86::BI__builtin_ia32_psllw512:
return interp__builtin_ia32_shift_with_count(
S, OpPC, Call,
[](const APInt &Elt, uint64_t Count) { return Elt.shl(Count); },
[](const APInt &Elt, unsigned Width) { return APInt::getZero(Width); });
case X86::BI__builtin_ia32_psrlq128:
case X86::BI__builtin_ia32_psrlq256:
case X86::BI__builtin_ia32_psrlq512:
case X86::BI__builtin_ia32_psrld128:
case X86::BI__builtin_ia32_psrld256:
case X86::BI__builtin_ia32_psrld512:
case X86::BI__builtin_ia32_psrlw128:
case X86::BI__builtin_ia32_psrlw256:
case X86::BI__builtin_ia32_psrlw512:
return interp__builtin_ia32_shift_with_count(
S, OpPC, Call,
[](const APInt &Elt, uint64_t Count) { return Elt.lshr(Count); },
[](const APInt &Elt, unsigned Width) { return APInt::getZero(Width); });
case X86::BI__builtin_ia32_pternlogd128_mask:
case X86::BI__builtin_ia32_pternlogd256_mask:
case X86::BI__builtin_ia32_pternlogd512_mask:
case X86::BI__builtin_ia32_pternlogq128_mask:
case X86::BI__builtin_ia32_pternlogq256_mask:
case X86::BI__builtin_ia32_pternlogq512_mask:
return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/false);
case X86::BI__builtin_ia32_pternlogd128_maskz:
case X86::BI__builtin_ia32_pternlogd256_maskz:
case X86::BI__builtin_ia32_pternlogd512_maskz:
case X86::BI__builtin_ia32_pternlogq128_maskz:
case X86::BI__builtin_ia32_pternlogq256_maskz:
case X86::BI__builtin_ia32_pternlogq512_maskz:
return interp__builtin_ia32_pternlog(S, OpPC, Call, /*MaskZ=*/true);
case Builtin::BI__builtin_elementwise_fshl:
return interp__builtin_elementwise_triop(S, OpPC, Call,
llvm::APIntOps::fshl);
case Builtin::BI__builtin_elementwise_fshr:
return interp__builtin_elementwise_triop(S, OpPC, Call,
llvm::APIntOps::fshr);
case X86::BI__builtin_ia32_shuf_f32x4_256:
case X86::BI__builtin_ia32_shuf_i32x4_256:
case X86::BI__builtin_ia32_shuf_f64x2_256:
case X86::BI__builtin_ia32_shuf_i64x2_256:
case X86::BI__builtin_ia32_shuf_f32x4:
case X86::BI__builtin_ia32_shuf_i32x4:
case X86::BI__builtin_ia32_shuf_f64x2:
case X86::BI__builtin_ia32_shuf_i64x2: {
// Destination and sources A, B all have the same type.
QualType VecQT = Call->getArg(0)->getType();
const auto *VecT = VecQT->castAs<VectorType>();
unsigned NumElems = VecT->getNumElements();
unsigned ElemBits = S.getASTContext().getTypeSize(VecT->getElementType());
unsigned LaneBits = 128u;
unsigned NumLanes = (NumElems * ElemBits) / LaneBits;
unsigned NumElemsPerLane = LaneBits / ElemBits;
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call,
[NumLanes, NumElemsPerLane](unsigned DstIdx, unsigned ShuffleMask) {
// DstIdx determines source. ShuffleMask selects lane in source.
unsigned BitsPerElem = NumLanes / 2;
unsigned IndexMask = (1u << BitsPerElem) - 1;
unsigned Lane = DstIdx / NumElemsPerLane;
unsigned SrcIdx = (Lane < NumLanes / 2) ? 0 : 1;
unsigned BitIdx = BitsPerElem * Lane;
unsigned SrcLaneIdx = (ShuffleMask >> BitIdx) & IndexMask;
unsigned ElemInLane = DstIdx % NumElemsPerLane;
unsigned IdxToPick = SrcLaneIdx * NumElemsPerLane + ElemInLane;
return std::pair<unsigned, int>{SrcIdx, IdxToPick};
});
}
case X86::BI__builtin_ia32_insertf32x4_256:
case X86::BI__builtin_ia32_inserti32x4_256:
case X86::BI__builtin_ia32_insertf64x2_256:
case X86::BI__builtin_ia32_inserti64x2_256:
case X86::BI__builtin_ia32_insertf32x4:
case X86::BI__builtin_ia32_inserti32x4:
case X86::BI__builtin_ia32_insertf64x2_512:
case X86::BI__builtin_ia32_inserti64x2_512:
case X86::BI__builtin_ia32_insertf32x8:
case X86::BI__builtin_ia32_inserti32x8:
case X86::BI__builtin_ia32_insertf64x4:
case X86::BI__builtin_ia32_inserti64x4:
case X86::BI__builtin_ia32_vinsertf128_ps256:
case X86::BI__builtin_ia32_vinsertf128_pd256:
case X86::BI__builtin_ia32_vinsertf128_si256:
case X86::BI__builtin_ia32_insert128i256:
return interp__builtin_x86_insert_subvector(S, OpPC, Call, BuiltinID);
case clang::X86::BI__builtin_ia32_vcvtps2ph:
case clang::X86::BI__builtin_ia32_vcvtps2ph256:
return interp__builtin_ia32_vcvtps2ph(S, OpPC, Call);
case X86::BI__builtin_ia32_vec_ext_v4hi:
case X86::BI__builtin_ia32_vec_ext_v16qi:
case X86::BI__builtin_ia32_vec_ext_v8hi:
case X86::BI__builtin_ia32_vec_ext_v4si:
case X86::BI__builtin_ia32_vec_ext_v2di:
case X86::BI__builtin_ia32_vec_ext_v32qi:
case X86::BI__builtin_ia32_vec_ext_v16hi:
case X86::BI__builtin_ia32_vec_ext_v8si:
case X86::BI__builtin_ia32_vec_ext_v4di:
case X86::BI__builtin_ia32_vec_ext_v4sf:
return interp__builtin_vec_ext(S, OpPC, Call, BuiltinID);
case X86::BI__builtin_ia32_vec_set_v4hi:
case X86::BI__builtin_ia32_vec_set_v16qi:
case X86::BI__builtin_ia32_vec_set_v8hi:
case X86::BI__builtin_ia32_vec_set_v4si:
case X86::BI__builtin_ia32_vec_set_v2di:
case X86::BI__builtin_ia32_vec_set_v32qi:
case X86::BI__builtin_ia32_vec_set_v16hi:
case X86::BI__builtin_ia32_vec_set_v8si:
case X86::BI__builtin_ia32_vec_set_v4di:
return interp__builtin_vec_set(S, OpPC, Call, BuiltinID);
case X86::BI__builtin_ia32_cvtb2mask128:
case X86::BI__builtin_ia32_cvtb2mask256:
case X86::BI__builtin_ia32_cvtb2mask512:
case X86::BI__builtin_ia32_cvtw2mask128:
case X86::BI__builtin_ia32_cvtw2mask256:
case X86::BI__builtin_ia32_cvtw2mask512:
case X86::BI__builtin_ia32_cvtd2mask128:
case X86::BI__builtin_ia32_cvtd2mask256:
case X86::BI__builtin_ia32_cvtd2mask512:
case X86::BI__builtin_ia32_cvtq2mask128:
case X86::BI__builtin_ia32_cvtq2mask256:
case X86::BI__builtin_ia32_cvtq2mask512:
return interp__builtin_ia32_cvt_vec2mask(S, OpPC, Call, BuiltinID);
case X86::BI__builtin_ia32_cmpb128_mask:
case X86::BI__builtin_ia32_cmpw128_mask:
case X86::BI__builtin_ia32_cmpd128_mask:
case X86::BI__builtin_ia32_cmpq128_mask:
case X86::BI__builtin_ia32_cmpb256_mask:
case X86::BI__builtin_ia32_cmpw256_mask:
case X86::BI__builtin_ia32_cmpd256_mask:
case X86::BI__builtin_ia32_cmpq256_mask:
case X86::BI__builtin_ia32_cmpb512_mask:
case X86::BI__builtin_ia32_cmpw512_mask:
case X86::BI__builtin_ia32_cmpd512_mask:
case X86::BI__builtin_ia32_cmpq512_mask:
return interp__builtin_ia32_cmp_mask(S, OpPC, Call, BuiltinID,
/*IsUnsigned=*/false);
case X86::BI__builtin_ia32_ucmpb128_mask:
case X86::BI__builtin_ia32_ucmpw128_mask:
case X86::BI__builtin_ia32_ucmpd128_mask:
case X86::BI__builtin_ia32_ucmpq128_mask:
case X86::BI__builtin_ia32_ucmpb256_mask:
case X86::BI__builtin_ia32_ucmpw256_mask:
case X86::BI__builtin_ia32_ucmpd256_mask:
case X86::BI__builtin_ia32_ucmpq256_mask:
case X86::BI__builtin_ia32_ucmpb512_mask:
case X86::BI__builtin_ia32_ucmpw512_mask:
case X86::BI__builtin_ia32_ucmpd512_mask:
case X86::BI__builtin_ia32_ucmpq512_mask:
return interp__builtin_ia32_cmp_mask(S, OpPC, Call, BuiltinID,
/*IsUnsigned=*/true);
case X86::BI__builtin_ia32_vpshufbitqmb128_mask:
case X86::BI__builtin_ia32_vpshufbitqmb256_mask:
case X86::BI__builtin_ia32_vpshufbitqmb512_mask:
return interp__builtin_ia32_shufbitqmb_mask(S, OpPC, Call);
case X86::BI__builtin_ia32_pslldqi128_byteshift:
case X86::BI__builtin_ia32_pslldqi256_byteshift:
case X86::BI__builtin_ia32_pslldqi512_byteshift:
// These SLLDQ intrinsics always operate on byte elements (8 bits).
// The lane width is hardcoded to 16 to match the SIMD register size,
// but the algorithm processes one byte per iteration,
// so APInt(8, ...) is correct and intentional.
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call,
[](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
unsigned LaneBase = (DstIdx / 16) * 16;
unsigned LaneIdx = DstIdx % 16;
if (LaneIdx < Shift)
return std::make_pair(0, -1);
return std::make_pair(0,
static_cast<int>(LaneBase + LaneIdx - Shift));
});
case X86::BI__builtin_ia32_psrldqi128_byteshift:
case X86::BI__builtin_ia32_psrldqi256_byteshift:
case X86::BI__builtin_ia32_psrldqi512_byteshift:
// These SRLDQ intrinsics always operate on byte elements (8 bits).
// The lane width is hardcoded to 16 to match the SIMD register size,
// but the algorithm processes one byte per iteration,
// so APInt(8, ...) is correct and intentional.
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call,
[](unsigned DstIdx, unsigned Shift) -> std::pair<unsigned, int> {
unsigned LaneBase = (DstIdx / 16) * 16;
unsigned LaneIdx = DstIdx % 16;
if (LaneIdx + Shift < 16)
return std::make_pair(0,
static_cast<int>(LaneBase + LaneIdx + Shift));
return std::make_pair(0, -1);
});
case X86::BI__builtin_ia32_palignr128:
case X86::BI__builtin_ia32_palignr256:
case X86::BI__builtin_ia32_palignr512:
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [](unsigned DstIdx, unsigned Shift) {
// Default to -1 → zero-fill this destination element
unsigned VecIdx = 1;
int ElemIdx = -1;
int Lane = DstIdx / 16;
int Offset = DstIdx % 16;
// Elements come from VecB first, then VecA after the shift boundary
unsigned ShiftedIdx = Offset + (Shift & 0xFF);
if (ShiftedIdx < 16) { // from VecB
ElemIdx = ShiftedIdx + (Lane * 16);
} else if (ShiftedIdx < 32) { // from VecA
VecIdx = 0;
ElemIdx = (ShiftedIdx - 16) + (Lane * 16);
}
return std::pair<unsigned, int>{VecIdx, ElemIdx};
});
case X86::BI__builtin_ia32_alignd128:
case X86::BI__builtin_ia32_alignd256:
case X86::BI__builtin_ia32_alignd512:
case X86::BI__builtin_ia32_alignq128:
case X86::BI__builtin_ia32_alignq256:
case X86::BI__builtin_ia32_alignq512: {
unsigned NumElems = Call->getType()->castAs<VectorType>()->getNumElements();
return interp__builtin_ia32_shuffle_generic(
S, OpPC, Call, [NumElems](unsigned DstIdx, unsigned Shift) {
unsigned Imm = Shift & 0xFF;
unsigned EffectiveShift = Imm & (NumElems - 1);
unsigned SourcePos = DstIdx + EffectiveShift;
unsigned VecIdx = SourcePos < NumElems ? 1u : 0u;
unsigned ElemIdx = SourcePos & (NumElems - 1);
return std::pair<unsigned, int>{VecIdx, static_cast<int>(ElemIdx)};
});
}
default:
S.FFDiag(S.Current->getLocation(OpPC),
diag::note_invalid_subexpr_in_const_expr)
<< S.Current->getRange(OpPC);
return false;
}
llvm_unreachable("Unhandled builtin ID");
}
bool InterpretOffsetOf(InterpState &S, CodePtr OpPC, const OffsetOfExpr *E,
ArrayRef<int64_t> ArrayIndices, int64_t &IntResult) {
CharUnits Result;
unsigned N = E->getNumComponents();
assert(N > 0);
unsigned ArrayIndex = 0;
QualType CurrentType = E->getTypeSourceInfo()->getType();
for (unsigned I = 0; I != N; ++I) {
const OffsetOfNode &Node = E->getComponent(I);
switch (Node.getKind()) {
case OffsetOfNode::Field: {
const FieldDecl *MemberDecl = Node.getField();
const auto *RD = CurrentType->getAsRecordDecl();
if (!RD || RD->isInvalidDecl())
return false;
const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(RD);
unsigned FieldIndex = MemberDecl->getFieldIndex();
assert(FieldIndex < RL.getFieldCount() && "offsetof field in wrong type");
Result +=
S.getASTContext().toCharUnitsFromBits(RL.getFieldOffset(FieldIndex));
CurrentType = MemberDecl->getType().getNonReferenceType();
break;
}
case OffsetOfNode::Array: {
// When generating bytecode, we put all the index expressions as Sint64 on
// the stack.
int64_t Index = ArrayIndices[ArrayIndex];
const ArrayType *AT = S.getASTContext().getAsArrayType(CurrentType);
if (!AT)
return false;
CurrentType = AT->getElementType();
CharUnits ElementSize = S.getASTContext().getTypeSizeInChars(CurrentType);
Result += Index * ElementSize;
++ArrayIndex;
break;
}
case OffsetOfNode::Base: {
const CXXBaseSpecifier *BaseSpec = Node.getBase();
if (BaseSpec->isVirtual())
return false;
// Find the layout of the class whose base we are looking into.
const auto *RD = CurrentType->getAsCXXRecordDecl();
if (!RD || RD->isInvalidDecl())
return false;
const ASTRecordLayout &RL = S.getASTContext().getASTRecordLayout(RD);
// Find the base class itself.
CurrentType = BaseSpec->getType();
const auto *BaseRD = CurrentType->getAsCXXRecordDecl();
if (!BaseRD)
return false;
// Add the offset to the base.
Result += RL.getBaseClassOffset(BaseRD);
break;
}
case OffsetOfNode::Identifier:
llvm_unreachable("Dependent OffsetOfExpr?");
}
}
IntResult = Result.getQuantity();
return true;
}
bool SetThreeWayComparisonField(InterpState &S, CodePtr OpPC,
const Pointer &Ptr, const APSInt &IntValue) {
const Record *R = Ptr.getRecord();
assert(R);
assert(R->getNumFields() == 1);
unsigned FieldOffset = R->getField(0u)->Offset;
const Pointer &FieldPtr = Ptr.atField(FieldOffset);
PrimType FieldT = *S.getContext().classify(FieldPtr.getType());
INT_TYPE_SWITCH(FieldT,
FieldPtr.deref<T>() = T::from(IntValue.getSExtValue()));
FieldPtr.initialize();
return true;
}
static void zeroAll(Pointer &Dest) {
const Descriptor *Desc = Dest.getFieldDesc();
if (Desc->isPrimitive()) {
TYPE_SWITCH(Desc->getPrimType(), {
Dest.deref<T>().~T();
new (&Dest.deref<T>()) T();
});
return;
}
if (Desc->isRecord()) {
const Record *R = Desc->ElemRecord;
for (const Record::Field &F : R->fields()) {
Pointer FieldPtr = Dest.atField(F.Offset);
zeroAll(FieldPtr);
}
return;
}
if (Desc->isPrimitiveArray()) {
for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
TYPE_SWITCH(Desc->getPrimType(), {
Dest.deref<T>().~T();
new (&Dest.deref<T>()) T();
});
}
return;
}
if (Desc->isCompositeArray()) {
for (unsigned I = 0, N = Desc->getNumElems(); I != N; ++I) {
Pointer ElemPtr = Dest.atIndex(I).narrow();
zeroAll(ElemPtr);
}
return;
}
}
static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,
Pointer &Dest, bool Activate);
static bool copyRecord(InterpState &S, CodePtr OpPC, const Pointer &Src,
Pointer &Dest, bool Activate = false) {
[[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
const Descriptor *DestDesc = Dest.getFieldDesc();
auto copyField = [&](const Record::Field &F, bool Activate) -> bool {
Pointer DestField = Dest.atField(F.Offset);
if (OptPrimType FT = S.Ctx.classify(F.Decl->getType())) {
TYPE_SWITCH(*FT, {
DestField.deref<T>() = Src.atField(F.Offset).deref<T>();
if (Src.atField(F.Offset).isInitialized())
DestField.initialize();
if (Activate)
DestField.activate();
});
return true;
}
// Composite field.
return copyComposite(S, OpPC, Src.atField(F.Offset), DestField, Activate);
};
assert(SrcDesc->isRecord());
assert(SrcDesc->ElemRecord == DestDesc->ElemRecord);
const Record *R = DestDesc->ElemRecord;
for (const Record::Field &F : R->fields()) {
if (R->isUnion()) {
// For unions, only copy the active field. Zero all others.
const Pointer &SrcField = Src.atField(F.Offset);
if (SrcField.isActive()) {
if (!copyField(F, /*Activate=*/true))
return false;
} else {
if (!CheckMutable(S, OpPC, Src.atField(F.Offset)))
return false;
Pointer DestField = Dest.atField(F.Offset);
zeroAll(DestField);
}
} else {
if (!copyField(F, Activate))
return false;
}
}
for (const Record::Base &B : R->bases()) {
Pointer DestBase = Dest.atField(B.Offset);
if (!copyRecord(S, OpPC, Src.atField(B.Offset), DestBase, Activate))
return false;
}
Dest.initialize();
return true;
}
static bool copyComposite(InterpState &S, CodePtr OpPC, const Pointer &Src,
Pointer &Dest, bool Activate = false) {
assert(Src.isLive() && Dest.isLive());
[[maybe_unused]] const Descriptor *SrcDesc = Src.getFieldDesc();
const Descriptor *DestDesc = Dest.getFieldDesc();
assert(!DestDesc->isPrimitive() && !SrcDesc->isPrimitive());
if (DestDesc->isPrimitiveArray()) {
assert(SrcDesc->isPrimitiveArray());
assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
PrimType ET = DestDesc->getPrimType();
for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
Pointer DestElem = Dest.atIndex(I);
TYPE_SWITCH(ET, {
DestElem.deref<T>() = Src.elem<T>(I);
DestElem.initialize();
});
}
return true;
}
if (DestDesc->isCompositeArray()) {
assert(SrcDesc->isCompositeArray());
assert(SrcDesc->getNumElems() == DestDesc->getNumElems());
for (unsigned I = 0, N = DestDesc->getNumElems(); I != N; ++I) {
const Pointer &SrcElem = Src.atIndex(I).narrow();
Pointer DestElem = Dest.atIndex(I).narrow();
if (!copyComposite(S, OpPC, SrcElem, DestElem, Activate))
return false;
}
return true;
}
if (DestDesc->isRecord())
return copyRecord(S, OpPC, Src, Dest, Activate);
return Invalid(S, OpPC);
}
bool DoMemcpy(InterpState &S, CodePtr OpPC, const Pointer &Src, Pointer &Dest) {
return copyComposite(S, OpPC, Src, Dest);
}
} // namespace interp
} // namespace clang
Reference in New Issue View Git Blame Copy Permalink