llvm-project/clang/lib/CIR/CodeGen/CIRGenExprCXX.cpp

//===--- CIRGenExprCXX.cpp - Emit CIR Code for C++ expressions ------------===//
//
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
// See https://llvm.org/LICENSE.txt for license information.
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
//
//===----------------------------------------------------------------------===//
//
// This contains code dealing with code generation of C++ expressions
//
//===----------------------------------------------------------------------===//

#include "CIRGenCXXABI.h"
#include "CIRGenConstantEmitter.h"
#include "CIRGenFunction.h"

#include "clang/AST/CharUnits.h"
#include "clang/AST/DeclCXX.h"
#include "clang/AST/ExprCXX.h"
#include "clang/AST/ExprObjC.h"
#include "clang/Basic/OperatorKinds.h"
#include "clang/CIR/MissingFeatures.h"
#include "llvm/Support/TrailingObjects.h"

using namespace clang;
using namespace clang::CIRGen;

namespace {
struct MemberCallInfo {
  RequiredArgs reqArgs;
  // Number of prefix arguments for the call. Ignores the `this` pointer.
  unsigned prefixSize;
};
} // namespace

static MemberCallInfo commonBuildCXXMemberOrOperatorCall(
    CIRGenFunction &cgf, const CXXMethodDecl *md, mlir::Value thisPtr,
    mlir::Value implicitParam, QualType implicitParamTy, const CallExpr *ce,
    CallArgList &args, CallArgList *rtlArgs) {
  assert(ce == nullptr || isa<CXXMemberCallExpr>(ce) ||
         isa<CXXOperatorCallExpr>(ce));
  assert(md->isInstance() &&
         "Trying to emit a member or operator call expr on a static method!");

  // Push the this ptr.
  const CXXRecordDecl *rd =
      cgf.cgm.getCXXABI().getThisArgumentTypeForMethod(md);
  args.add(RValue::get(thisPtr), cgf.getTypes().deriveThisType(rd, md));

  // If there is an implicit parameter (e.g. VTT), emit it.
  if (implicitParam) {
    args.add(RValue::get(implicitParam), implicitParamTy);
  }

  const auto *fpt = md->getType()->castAs<FunctionProtoType>();
  RequiredArgs required =
      RequiredArgs::getFromProtoWithExtraSlots(fpt, args.size());
  unsigned prefixSize = args.size() - 1;

  // Add the rest of the call args
  if (rtlArgs) {
    // Special case: if the caller emitted the arguments right-to-left already
    // (prior to emitting the *this argument), we're done. This happens for
    // assignment operators.
    args.addFrom(*rtlArgs);
  } else if (ce) {
    // Special case: skip first argument of CXXOperatorCall (it is "this").
    unsigned argsToSkip = isa<CXXOperatorCallExpr>(ce) ? 1 : 0;
    cgf.emitCallArgs(args, fpt, drop_begin(ce->arguments(), argsToSkip),
                     ce->getDirectCallee());
  } else {
    assert(
        fpt->getNumParams() == 0 &&
        "No CallExpr specified for function with non-zero number of arguments");
  }

  //  return {required, prefixSize};
  return {required, prefixSize};
}

RValue
CIRGenFunction::emitCXXMemberPointerCallExpr(const CXXMemberCallExpr *ce,
                                             ReturnValueSlot returnValue) {
  const BinaryOperator *bo =
      cast<BinaryOperator>(ce->getCallee()->IgnoreParens());
  const Expr *baseExpr = bo->getLHS();
  const Expr *memFnExpr = bo->getRHS();

  const auto *mpt = memFnExpr->getType()->castAs<MemberPointerType>();
  const auto *fpt = mpt->getPointeeType()->castAs<FunctionProtoType>();

  // Emit the 'this' pointer.
  Address thisAddr = Address::invalid();
  if (bo->getOpcode() == BO_PtrMemI)
    thisAddr = emitPointerWithAlignment(baseExpr);
  else
    thisAddr = emitLValue(baseExpr).getAddress();

  assert(!cir::MissingFeatures::emitTypeCheck());

  // Get the member function pointer.
  mlir::Value memFnPtr = emitScalarExpr(memFnExpr);

  // Resolve the member function pointer to the actual callee and adjust the
  // "this" pointer for call.
  mlir::Location loc = getLoc(ce->getExprLoc());
  auto [/*mlir::Value*/ calleePtr, /*mlir::Value*/ adjustedThis] =
      builder.createGetMethod(loc, memFnPtr, thisAddr.getPointer());

  // Prepare the call arguments.
  CallArgList argsList;
  argsList.add(RValue::get(adjustedThis), getContext().VoidPtrTy);
  emitCallArgs(argsList, fpt, ce->arguments());

  RequiredArgs required = RequiredArgs::getFromProtoWithExtraSlots(fpt, 1);

  // Build the call.
  CIRGenCallee callee(fpt, calleePtr.getDefiningOp());
  assert(!cir::MissingFeatures::opCallMustTail());
  return emitCall(cgm.getTypes().arrangeCXXMethodCall(argsList, fpt, required,
                                                      /*PrefixSize=*/0),
                  callee, returnValue, argsList, nullptr, loc);
}

RValue CIRGenFunction::emitCXXMemberOrOperatorMemberCallExpr(
    const CallExpr *ce, const CXXMethodDecl *md, ReturnValueSlot returnValue,
    bool hasQualifier, NestedNameSpecifier qualifier, bool isArrow,
    const Expr *base) {
  assert(isa<CXXMemberCallExpr>(ce) || isa<CXXOperatorCallExpr>(ce));

  // Compute the object pointer.
  bool canUseVirtualCall = md->isVirtual() && !hasQualifier;
  const CXXMethodDecl *devirtualizedMethod = nullptr;
  assert(!cir::MissingFeatures::devirtualizeMemberFunction());

  // Note on trivial assignment
  // --------------------------
  // Classic codegen avoids generating the trivial copy/move assignment operator
  // when it isn't necessary, choosing instead to just produce IR with an
  // equivalent effect. We have chosen not to do that in CIR, instead emitting
  // trivial copy/move assignment operators and allowing later transformations
  // to optimize them away if appropriate.

  // C++17 demands that we evaluate the RHS of a (possibly-compound) assignment
  // operator before the LHS.
  CallArgList rtlArgStorage;
  CallArgList *rtlArgs = nullptr;
  if (auto *oce = dyn_cast<CXXOperatorCallExpr>(ce)) {
    if (oce->isAssignmentOp()) {
      rtlArgs = &rtlArgStorage;
      emitCallArgs(*rtlArgs, md->getType()->castAs<FunctionProtoType>(),
                   drop_begin(ce->arguments(), 1), ce->getDirectCallee(),
                   /*ParamsToSkip*/ 0);
    }
  }

  LValue thisPtr;
  if (isArrow) {
    LValueBaseInfo baseInfo;
    assert(!cir::MissingFeatures::opTBAA());
    Address thisValue = emitPointerWithAlignment(base, &baseInfo);
    thisPtr = makeAddrLValue(thisValue, base->getType(), baseInfo);
  } else {
    thisPtr = emitLValue(base);
  }

  if (isa<CXXConstructorDecl>(md)) {
    cgm.errorNYI(ce->getSourceRange(),
                 "emitCXXMemberOrOperatorMemberCallExpr: constructor call");
    return RValue::get(nullptr);
  }

  if ((md->isTrivial() || (md->isDefaulted() && md->getParent()->isUnion())) &&
      isa<CXXDestructorDecl>(md))
    return RValue::get(nullptr);

  // Compute the function type we're calling
  const CXXMethodDecl *calleeDecl =
      devirtualizedMethod ? devirtualizedMethod : md;
  const CIRGenFunctionInfo *fInfo = nullptr;
  if (const auto *dtor = dyn_cast<CXXDestructorDecl>(calleeDecl))
    fInfo = &cgm.getTypes().arrangeCXXStructorDeclaration(
        GlobalDecl(dtor, Dtor_Complete));
  else
    fInfo = &cgm.getTypes().arrangeCXXMethodDeclaration(calleeDecl);

  cir::FuncType ty = cgm.getTypes().getFunctionType(*fInfo);

  assert(!cir::MissingFeatures::sanitizers());
  assert(!cir::MissingFeatures::emitTypeCheck());

  // C++ [class.virtual]p12:
  //   Explicit qualification with the scope operator (5.1) suppresses the
  //   virtual call mechanism.
  //
  // We also don't emit a virtual call if the base expression has a record type
  // because then we know what the type is.
  bool useVirtualCall = canUseVirtualCall && !devirtualizedMethod;

  if (const auto *dtor = dyn_cast<CXXDestructorDecl>(calleeDecl)) {
    assert(ce->arg_begin() == ce->arg_end() &&
           "Destructor shouldn't have explicit parameters");
    assert(returnValue.isNull() && "Destructor shouldn't have return value");
    if (useVirtualCall) {
      cgm.getCXXABI().emitVirtualDestructorCall(*this, dtor, Dtor_Complete,
                                                thisPtr.getAddress(),
                                                cast<CXXMemberCallExpr>(ce));
    } else {
      GlobalDecl globalDecl(dtor, Dtor_Complete);
      CIRGenCallee callee;
      assert(!cir::MissingFeatures::appleKext());
      if (!devirtualizedMethod) {
        callee = CIRGenCallee::forDirect(
            cgm.getAddrOfCXXStructor(globalDecl, fInfo, ty), globalDecl);
      } else {
        cgm.errorNYI(ce->getSourceRange(), "devirtualized destructor call");
        return RValue::get(nullptr);
      }

      QualType thisTy =
          isArrow ? base->getType()->getPointeeType() : base->getType();
      // CIRGen does not pass CallOrInvoke here (different from OG LLVM codegen)
      // because in practice it always null even in OG.
      emitCXXDestructorCall(globalDecl, callee, thisPtr.getPointer(), thisTy,
                            /*implicitParam=*/nullptr,
                            /*implicitParamTy=*/QualType(), ce);
    }
    return RValue::get(nullptr);
  }

  CIRGenCallee callee;
  if (useVirtualCall) {
    callee = CIRGenCallee::forVirtual(ce, md, thisPtr.getAddress(), ty);
  } else {
    assert(!cir::MissingFeatures::sanitizers());
    if (getLangOpts().AppleKext) {
      cgm.errorNYI(ce->getSourceRange(),
                   "emitCXXMemberOrOperatorMemberCallExpr: AppleKext");
      return RValue::get(nullptr);
    }

    callee = CIRGenCallee::forDirect(cgm.getAddrOfFunction(calleeDecl, ty),
                                     GlobalDecl(calleeDecl));
  }

  if (md->isVirtual()) {
    Address newThisAddr =
        cgm.getCXXABI().adjustThisArgumentForVirtualFunctionCall(
            *this, calleeDecl, thisPtr.getAddress(), useVirtualCall);
    thisPtr.setAddress(newThisAddr);
  }

  return emitCXXMemberOrOperatorCall(
      calleeDecl, callee, returnValue, thisPtr.getPointer(),
      /*ImplicitParam=*/nullptr, QualType(), ce, rtlArgs);
}

RValue
CIRGenFunction::emitCXXOperatorMemberCallExpr(const CXXOperatorCallExpr *e,
                                              const CXXMethodDecl *md,
                                              ReturnValueSlot returnValue) {
  assert(md->isInstance() &&
         "Trying to emit a member call expr on a static method!");
  return emitCXXMemberOrOperatorMemberCallExpr(
      e, md, returnValue, /*HasQualifier=*/false, /*Qualifier=*/std::nullopt,
      /*IsArrow=*/false, e->getArg(0));
}

RValue CIRGenFunction::emitCUDAKernelCallExpr(const CUDAKernelCallExpr *expr,
                                              ReturnValueSlot returnValue) {
  // Emit as a device kernel call if CUDA device code is to be generated.
  if (!getLangOpts().HIP && getLangOpts().CUDAIsDevice)
    cgm.errorNYI("CUDA Device side kernel call");
  return cgm.getCUDARuntime().emitCUDAKernelCallExpr(*this, expr, returnValue);
}

RValue CIRGenFunction::emitCXXMemberOrOperatorCall(
    const CXXMethodDecl *md, const CIRGenCallee &callee,
    ReturnValueSlot returnValue, mlir::Value thisPtr, mlir::Value implicitParam,
    QualType implicitParamTy, const CallExpr *ce, CallArgList *rtlArgs) {
  const auto *fpt = md->getType()->castAs<FunctionProtoType>();
  CallArgList args;
  MemberCallInfo callInfo = commonBuildCXXMemberOrOperatorCall(
      *this, md, thisPtr, implicitParam, implicitParamTy, ce, args, rtlArgs);
  auto &fnInfo = cgm.getTypes().arrangeCXXMethodCall(
      args, fpt, callInfo.reqArgs, callInfo.prefixSize);
  assert((ce || currSrcLoc) && "expected source location");
  mlir::Location loc = ce ? getLoc(ce->getExprLoc()) : *currSrcLoc;
  assert(!cir::MissingFeatures::opCallMustTail());
  return emitCall(fnInfo, callee, returnValue, args, nullptr, loc);
}

static void emitNullBaseClassInitialization(CIRGenFunction &cgf,
                                            Address destPtr,
                                            const CXXRecordDecl *base) {
  if (base->isEmpty())
    return;

  const ASTRecordLayout &layout = cgf.getContext().getASTRecordLayout(base);
  CharUnits nvSize = layout.getNonVirtualSize();

  // We cannot simply zero-initialize the entire base sub-object if vbptrs are
  // present, they are initialized by the most derived class before calling the
  // constructor.
  SmallVector<std::pair<CharUnits, CharUnits>, 1> stores;
  stores.emplace_back(CharUnits::Zero(), nvSize);

  // Each store is split by the existence of a vbptr.
  // TODO(cir): This only needs handling for the MS CXXABI.
  assert(!cir::MissingFeatures::msabi());

  // If the type contains a pointer to data member we can't memset it to zero.
  // Instead, create a null constant and copy it to the destination.
  // TODO: there are other patterns besides zero that we can usefully memset,
  // like -1, which happens to be the pattern used by member-pointers.
  // TODO: isZeroInitializable can be over-conservative in the case where a
  // virtual base contains a member pointer.
  mlir::TypedAttr nullConstantForBase = cgf.cgm.emitNullConstantForBase(base);
  if (!cgf.getBuilder().isNullValue(nullConstantForBase)) {
    cgf.cgm.errorNYI(
        base->getSourceRange(),
        "emitNullBaseClassInitialization: base constant is not null");
  } else {
    // Otherwise, just memset the whole thing to zero.  This is legal
    // because in LLVM, all default initializers (other than the ones we just
    // handled above) are guaranteed to have a bit pattern of all zeros.
    // TODO(cir): When the MS CXXABI is supported, we will need to iterate over
    // `stores` and create a separate memset for each one. For now, we know that
    // there will only be one store and it will begin at offset zero, so that
    // simplifies this code considerably.
    assert(stores.size() == 1 && "Expected only one store");
    assert(stores[0].first == CharUnits::Zero() &&
           "Expected store to begin at offset zero");
    CIRGenBuilderTy builder = cgf.getBuilder();
    mlir::Location loc = cgf.getLoc(base->getBeginLoc());
    builder.createStore(loc, builder.getConstant(loc, nullConstantForBase),
                        destPtr);
  }
}

void CIRGenFunction::emitCXXConstructExpr(const CXXConstructExpr *e,
                                          AggValueSlot dest) {
  assert(!dest.isIgnored() && "Must have a destination!");
  const CXXConstructorDecl *cd = e->getConstructor();

  // If we require zero initialization before (or instead of) calling the
  // constructor, as can be the case with a non-user-provided default
  // constructor, emit the zero initialization now, unless destination is
  // already zeroed.
  if (e->requiresZeroInitialization() && !dest.isZeroed()) {
    switch (e->getConstructionKind()) {
    case CXXConstructionKind::Delegating:
    case CXXConstructionKind::Complete:
      emitNullInitialization(getLoc(e->getSourceRange()), dest.getAddress(),
                             e->getType());
      break;
    case CXXConstructionKind::VirtualBase:
    case CXXConstructionKind::NonVirtualBase:
      emitNullBaseClassInitialization(*this, dest.getAddress(),
                                      cd->getParent());
      break;
    }
  }

  // If this is a call to a trivial default constructor, do nothing.
  if (cd->isTrivial() && cd->isDefaultConstructor())
    return;

  // Elide the constructor if we're constructing from a temporary
  if (getLangOpts().ElideConstructors && e->isElidable()) {
    // FIXME: This only handles the simplest case, where the source object is
    //        passed directly as the first argument to the constructor. This
    //        should also handle stepping through implicit casts and conversion
    //        sequences which involve two steps, with a conversion operator
    //        follwed by a converting constructor.
    const Expr *srcObj = e->getArg(0);
    assert(srcObj->isTemporaryObject(getContext(), cd->getParent()));
    assert(
        getContext().hasSameUnqualifiedType(e->getType(), srcObj->getType()));
    emitAggExpr(srcObj, dest);
    return;
  }

  if (const ArrayType *arrayType = getContext().getAsArrayType(e->getType())) {
    assert(!cir::MissingFeatures::sanitizers());
    emitCXXAggrConstructorCall(cd, arrayType, dest.getAddress(), e, false);
  } else {

    clang::CXXCtorType type = Ctor_Complete;
    bool forVirtualBase = false;
    bool delegating = false;

    switch (e->getConstructionKind()) {
    case CXXConstructionKind::Complete:
      type = Ctor_Complete;
      break;
    case CXXConstructionKind::Delegating:
      // We should be emitting a constructor; GlobalDecl will assert this
      type = curGD.getCtorType();
      delegating = true;
      break;
    case CXXConstructionKind::VirtualBase:
      forVirtualBase = true;
      [[fallthrough]];
    case CXXConstructionKind::NonVirtualBase:
      type = Ctor_Base;
      break;
    }

    emitCXXConstructorCall(cd, type, forVirtualBase, delegating, dest, e);
  }
}

static CharUnits calculateCookiePadding(CIRGenFunction &cgf,
                                        const CXXNewExpr *e) {
  if (!e->isArray())
    return CharUnits::Zero();

  // No cookie is required if the operator new[] being used is the
  // reserved placement operator new[].
  if (e->getOperatorNew()->isReservedGlobalPlacementOperator())
    return CharUnits::Zero();

  return cgf.cgm.getCXXABI().getArrayCookieSize(e);
}

static mlir::Value emitCXXNewAllocSize(CIRGenFunction &cgf, const CXXNewExpr *e,
                                       unsigned minElements,
                                       mlir::Value &numElements,
                                       mlir::Value &sizeWithoutCookie) {
  QualType type = e->getAllocatedType();
  mlir::Location loc = cgf.getLoc(e->getSourceRange());

  if (!e->isArray()) {
    CharUnits typeSize = cgf.getContext().getTypeSizeInChars(type);
    sizeWithoutCookie = cgf.getBuilder().getConstant(
        loc, cir::IntAttr::get(cgf.sizeTy, typeSize.getQuantity()));
    return sizeWithoutCookie;
  }

  // The width of size_t.
  unsigned sizeWidth = cgf.cgm.getDataLayout().getTypeSizeInBits(cgf.sizeTy);

  // The number of elements can be have an arbitrary integer type;
  // essentially, we need to multiply it by a constant factor, add a
  // cookie size, and verify that the result is representable as a
  // size_t.  That's just a gloss, though, and it's wrong in one
  // important way: if the count is negative, it's an error even if
  // the cookie size would bring the total size >= 0.
  //
  // If the array size is constant, Sema will have prevented negative
  // values and size overflow.

  // Compute the constant factor.
  llvm::APInt arraySizeMultiplier(sizeWidth, 1);
  while (const ConstantArrayType *cat =
             cgf.getContext().getAsConstantArrayType(type)) {
    type = cat->getElementType();
    arraySizeMultiplier *= cat->getSize();
  }

  CharUnits typeSize = cgf.getContext().getTypeSizeInChars(type);
  llvm::APInt typeSizeMultiplier(sizeWidth, typeSize.getQuantity());
  typeSizeMultiplier *= arraySizeMultiplier;

  // Figure out the cookie size.
  llvm::APInt cookieSize(sizeWidth,
                         calculateCookiePadding(cgf, e).getQuantity());

  // This will be a size_t.
  mlir::Value size;

  // Emit the array size expression.
  // We multiply the size of all dimensions for NumElements.
  // e.g for 'int[2][3]', ElemType is 'int' and NumElements is 6.
  const Expr *arraySize = *e->getArraySize();
  mlir::Attribute constNumElements =
      ConstantEmitter(cgf.cgm, &cgf)
          .tryEmitAbstract(arraySize, arraySize->getType());
  if (constNumElements) {
    // Get an APInt from the constant
    const llvm::APInt &count =
        mlir::cast<cir::IntAttr>(constNumElements).getValue();

    [[maybe_unused]] unsigned numElementsWidth = count.getBitWidth();
    bool hasAnyOverflow = false;

    // The equivalent code in CodeGen/CGExprCXX.cpp handles these cases as
    // overflow, but that should never happen. The size argument is implicitly
    // cast to a size_t, so it can never be negative and numElementsWidth will
    // always equal sizeWidth.
    assert(!count.isNegative() && "Expected non-negative array size");
    assert(numElementsWidth == sizeWidth &&
           "Expected a size_t array size constant");

    // Okay, compute a count at the right width.
    llvm::APInt adjustedCount = count.zextOrTrunc(sizeWidth);

    // Scale numElements by that.  This might overflow, but we don't
    // care because it only overflows if allocationSize does too, and
    // if that overflows then we shouldn't use this.
    // This emits a constant that may not be used, but we can't tell here
    // whether it will be needed or not.
    numElements =
        cgf.getBuilder().getConstInt(loc, adjustedCount * arraySizeMultiplier);

    // Compute the size before cookie, and track whether it overflowed.
    bool overflow;
    llvm::APInt allocationSize =
        adjustedCount.umul_ov(typeSizeMultiplier, overflow);

    // Sema prevents us from hitting this case
    assert(!overflow && "Overflow in array allocation size");

    // Add in the cookie, and check whether it's overflowed.
    if (cookieSize != 0) {
      // Save the current size without a cookie.  This shouldn't be
      // used if there was overflow
      sizeWithoutCookie = cgf.getBuilder().getConstInt(
          loc, allocationSize.zextOrTrunc(sizeWidth));

      allocationSize = allocationSize.uadd_ov(cookieSize, overflow);
      hasAnyOverflow |= overflow;
    }

    // On overflow, produce a -1 so operator new will fail
    if (hasAnyOverflow) {
      size =
          cgf.getBuilder().getConstInt(loc, llvm::APInt::getAllOnes(sizeWidth));
    } else {
      size = cgf.getBuilder().getConstInt(loc, allocationSize);
    }
  } else {
    // Create a value for the variable number of elements
    numElements = cgf.emitScalarExpr(*e->getArraySize());
    auto numElementsType = mlir::cast<cir::IntType>(numElements.getType());
    unsigned numElementsWidth = numElementsType.getWidth();

    // The number of elements can have an arbitrary integer type;
    // essentially, we need to multiply it by a constant factor, add a
    // cookie size, and verify that the result is representable as a
    // size_t.  That's just a gloss, though, and it's wrong in one
    // important way: if the count is negative, it's an error even if
    // the cookie size would bring the total size >= 0.
    bool isSigned =
        (*e->getArraySize())->getType()->isSignedIntegerOrEnumerationType();

    // There are up to five conditions we need to test for:
    // 1) if isSigned, we need to check whether numElements is negative;
    // 2) if numElementsWidth > sizeWidth, we need to check whether
    //    numElements is larger than something representable in size_t;
    // 3) if minElements > 0, we need to check whether numElements is smaller
    //    than that.
    // 4) we need to compute
    //      sizeWithoutCookie := numElements * typeSizeMultiplier
    //    and check whether it overflows; and
    // 5) if we need a cookie, we need to compute
    //      size := sizeWithoutCookie + cookieSize
    //    and check whether it overflows.

    mlir::Value hasOverflow;

    // If numElementsWidth > sizeWidth, then one way or another, we're
    // going to have to do a comparison for (2), and this happens to
    // take care of (1), too.
    if (numElementsWidth > sizeWidth) {
      llvm::APInt threshold =
          llvm::APInt::getOneBitSet(numElementsWidth, sizeWidth);

      // Use an unsigned comparison regardless of the sign of numElements.
      mlir::Value unsignedNumElements = numElements;
      if (isSigned)
        unsignedNumElements = cgf.getBuilder().createIntCast(
            numElements, cgf.getBuilder().getUIntNTy(numElementsWidth));

      mlir::Value thresholdV =
          cgf.getBuilder().getConstInt(loc, threshold, /*isUnsigned=*/true);
      hasOverflow = cgf.getBuilder().createCompare(
          loc, cir::CmpOpKind::ge, unsignedNumElements, thresholdV);
      numElements = cgf.getBuilder().createIntCast(
          unsignedNumElements, mlir::cast<cir::IntType>(cgf.sizeTy));

      // Otherwise, if we're signed, we want to sext up to size_t.
    } else if (isSigned) {
      if (numElementsWidth < sizeWidth)
        numElements = cgf.getBuilder().createIntCast(
            numElements, cgf.getBuilder().getSIntNTy(sizeWidth));

      // If there's a non-1 type size multiplier, then we can do the
      // signedness check at the same time as we do the multiply
      // because a negative number times anything will cause an
      // unsigned overflow.  Otherwise, we have to do it here. But at
      // least in this case, we can subsume the >= minElements check.
      if (typeSizeMultiplier == 1)
        hasOverflow = cgf.getBuilder().createCompare(
            loc, cir::CmpOpKind::lt, numElements,
            cgf.getBuilder().getConstInt(loc, numElements.getType(),
                                         minElements));

      numElements = cgf.getBuilder().createIntCast(
          numElements, mlir::cast<cir::IntType>(cgf.sizeTy));

      // Otherwise, zext up to size_t if necessary.
    } else if (numElementsWidth < sizeWidth) {
      numElements = cgf.getBuilder().createIntCast(
          numElements, mlir::cast<cir::IntType>(cgf.sizeTy));
    }

    assert(numElements.getType() == cgf.sizeTy);

    if (minElements) {
      // Don't allow allocation of fewer elements than we have initializers.
      if (!hasOverflow) {
        mlir::Value minElementsV = cgf.getBuilder().getConstInt(
            loc, llvm::APInt(sizeWidth, minElements));
        hasOverflow = cgf.getBuilder().createCompare(loc, cir::CmpOpKind::lt,
                                                     numElements, minElementsV);
      } else if (numElementsWidth > sizeWidth) {
        // The other existing overflow subsumes this check.
        // We do an unsigned comparison, since any signed value < -1 is
        // taken care of either above or below.
        mlir::Value minElementsV = cgf.getBuilder().getConstInt(
            loc, llvm::APInt(sizeWidth, minElements));
        hasOverflow = cgf.getBuilder().createOr(
            loc, hasOverflow,
            cgf.getBuilder().createCompare(loc, cir::CmpOpKind::lt, numElements,
                                           minElementsV));
      }
    }

    size = numElements;

    // Multiply by the type size if necessary.  This multiplier
    // includes all the factors for nested arrays.
    //
    // This step also causes numElements to be scaled up by the
    // nested-array factor if necessary.  Overflow on this computation
    // can be ignored because the result shouldn't be used if
    // allocation fails.
    if (typeSizeMultiplier != 1) {
      mlir::Value tsmV = cgf.getBuilder().getConstInt(loc, typeSizeMultiplier);
      auto mulOp = cir::MulOverflowOp::create(
          cgf.getBuilder(), loc, mlir::cast<cir::IntType>(cgf.sizeTy), size,
          tsmV);

      if (hasOverflow)
        hasOverflow =
            cgf.getBuilder().createOr(loc, hasOverflow, mulOp.getOverflow());
      else
        hasOverflow = mulOp.getOverflow();

      size = mulOp.getResult();

      // Also scale up numElements by the array size multiplier.
      if (arraySizeMultiplier != 1) {
        // If the base element type size is 1, then we can re-use the
        // multiply we just did.
        if (typeSize.isOne()) {
          assert(arraySizeMultiplier == typeSizeMultiplier);
          numElements = size;

          // Otherwise we need a separate multiply.
        } else {
          mlir::Value asmV =
              cgf.getBuilder().getConstInt(loc, arraySizeMultiplier);
          numElements = cgf.getBuilder().createMul(loc, numElements, asmV);
        }
      }
    } else {
      // numElements doesn't need to be scaled.
      assert(arraySizeMultiplier == 1);
    }

    // Add in the cookie size if necessary.
    if (cookieSize != 0) {
      sizeWithoutCookie = size;
      mlir::Value cookieSizeV = cgf.getBuilder().getConstInt(loc, cookieSize);
      auto addOp = cir::AddOverflowOp::create(
          cgf.getBuilder(), loc, mlir::cast<cir::IntType>(cgf.sizeTy), size,
          cookieSizeV);

      if (hasOverflow)
        hasOverflow =
            cgf.getBuilder().createOr(loc, hasOverflow, addOp.getOverflow());
      else
        hasOverflow = addOp.getOverflow();

      size = addOp.getResult();
    }

    // If we had any possibility of dynamic overflow, make a select to
    // overwrite 'size' with an all-ones value, which should cause
    // operator new to throw.
    if (hasOverflow) {
      mlir::Value allOnes =
          cgf.getBuilder().getConstInt(loc, llvm::APInt::getAllOnes(sizeWidth));
      size = cgf.getBuilder().createSelect(loc, hasOverflow, allOnes, size);
    }
  }

  if (cookieSize == 0)
    sizeWithoutCookie = size;
  else
    assert(sizeWithoutCookie && "didn't set sizeWithoutCookie?");

  return size;
}

/// Emit a call to an operator new or operator delete function, as implicitly
/// created by new-expressions and delete-expressions.
static RValue emitNewDeleteCall(CIRGenFunction &cgf,
                                const FunctionDecl *calleeDecl,
                                const FunctionProtoType *calleeType,
                                const CallArgList &args) {
  cir::CIRCallOpInterface callOrTryCall;
  cir::FuncOp calleePtr = cgf.cgm.getAddrOfFunction(calleeDecl);
  CIRGenCallee callee =
      CIRGenCallee::forDirect(calleePtr, GlobalDecl(calleeDecl));
  RValue rv =
      cgf.emitCall(cgf.cgm.getTypes().arrangeFreeFunctionCall(args, calleeType),
                   callee, ReturnValueSlot(), args, &callOrTryCall);

  /// C++1y [expr.new]p10:
  ///   [In a new-expression,] an implementation is allowed to omit a call
  ///   to a replaceable global allocation function.
  ///
  /// We model such elidable calls with the 'builtin' attribute.
  if (calleeDecl->isReplaceableGlobalAllocationFunction() && calleePtr &&
      calleePtr->hasAttr(cir::CIRDialect::getNoBuiltinAttrName())) {
    callOrTryCall->setAttr(cir::CIRDialect::getBuiltinAttrName(),
                           mlir::UnitAttr::get(callOrTryCall->getContext()));
  }

  return rv;
}

RValue CIRGenFunction::emitNewOrDeleteBuiltinCall(const FunctionProtoType *type,
                                                  const CallExpr *callExpr,
                                                  OverloadedOperatorKind op) {
  CallArgList args;
  emitCallArgs(args, type, callExpr->arguments());
  // Find the allocation or deallocation function that we're calling.
  ASTContext &astContext = getContext();
  assert(op == OO_New || op == OO_Delete);
  DeclarationName name = astContext.DeclarationNames.getCXXOperatorName(op);

  clang::DeclContextLookupResult lookupResult =
      astContext.getTranslationUnitDecl()->lookup(name);
  for (const NamedDecl *decl : lookupResult) {
    if (const auto *funcDecl = dyn_cast<FunctionDecl>(decl)) {
      if (astContext.hasSameType(funcDecl->getType().getTypePtr(), type)) {
        if (sanOpts.has(SanitizerKind::AllocToken)) {
          // TODO: Set !alloc_token metadata.
          assert(!cir::MissingFeatures::allocToken());
          cgm.errorNYI("Alloc token sanitizer not yet supported!");
        }

        // Emit the call to operator new/delete.
        return emitNewDeleteCall(*this, funcDecl, type, args);
      }
    }
  }

  llvm_unreachable("predeclared global operator new/delete is missing");
}

namespace {
template <typename Traits> struct PlacementArg {
  typename Traits::RValueTy argValue;
  QualType argType;
};

/// A cleanup to call the given 'operator delete' function upon abnormal
/// exit from a new expression. Templated on a traits type that deals with
/// ensuring that the arguments dominate the cleanup if necessary.
template <typename Traits>
class CallDeleteDuringNew final
    : public EHScopeStack::Cleanup,
      private llvm::TrailingObjects<CallDeleteDuringNew<Traits>,
                                    PlacementArg<Traits>> {
  using TrailingObj =
      llvm::TrailingObjects<CallDeleteDuringNew<Traits>, PlacementArg<Traits>>;
  friend TrailingObj;
  using TrailingObj::getTrailingObjects;

  /// Type used to hold llvm::Value*s.
  typedef typename Traits::ValueTy ValueTy;
  /// Type used to hold RValues.
  typedef typename Traits::RValueTy RValueTy;

  unsigned numPlacementArgs : 30;
  LLVM_PREFERRED_TYPE(AlignedAllocationMode)
  unsigned passAlignmentToPlacementDelete : 1;
  const FunctionDecl *operatorDelete;
  ValueTy ptr;
  ValueTy allocSize;
  CharUnits allocAlign;

  PlacementArg<Traits> *getPlacementArgs() { return getTrailingObjects(); }

  void setPlacementArg(unsigned i, RValueTy argValue, QualType argType) {
    assert(i < numPlacementArgs && "index out of range");
    getPlacementArgs()[i] = {argValue, argType};
  }

public:
  static size_t getExtraSize(size_t numPlacementArgs) {
    return TrailingObj::template additionalSizeToAlloc<PlacementArg<Traits>>(
        numPlacementArgs);
  }

  CallDeleteDuringNew(size_t numPlacementArgs,
                      const FunctionDecl *operatorDelete, ValueTy ptr,
                      ValueTy allocSize,
                      const ImplicitAllocationParameters &iap,
                      CharUnits allocAlign, const CallArgList *newArgs,
                      unsigned numNonPlacementArgs, CIRGenFunction *cgf,
                      mlir::Location loc)
      : numPlacementArgs(numPlacementArgs),
        passAlignmentToPlacementDelete(isAlignedAllocation(iap.PassAlignment)),
        operatorDelete(operatorDelete), ptr(ptr), allocSize(allocSize),
        allocAlign(allocAlign) {
    for (unsigned i = 0, n = numPlacementArgs; i != n; ++i) {
      const CallArg &arg = (*newArgs)[i + numNonPlacementArgs];
      setPlacementArg(i, arg.getRValue(*cgf, loc), arg.ty);
    }
  }

  void emit(CIRGenFunction &cgf, Flags flags) override {
    const auto *fpt = operatorDelete->getType()->castAs<FunctionProtoType>();
    CallArgList deleteArgs;

    unsigned firstNonTypeArg = 0;
    TypeAwareAllocationMode typeAwareDeallocation = TypeAwareAllocationMode::No;
    assert(!cir::MissingFeatures::typeAwareAllocation());

    // The first argument after type-identity parameter (if any) is always
    // a void* (or C* for a destroying operator delete for class type C).
    deleteArgs.add(Traits::get(cgf, ptr), fpt->getParamType(firstNonTypeArg));

    // Figure out what other parameters we should be implicitly passing.
    UsualDeleteParams params;
    if (numPlacementArgs) {
      // A placement deallocation function is implicitly passed an alignment
      // if the placement allocation function was, but is never passed a size.
      params.Alignment =
          alignedAllocationModeFromBool(passAlignmentToPlacementDelete);
      params.TypeAwareDelete = typeAwareDeallocation;
      params.Size = isTypeAwareAllocation(params.TypeAwareDelete);
    } else {
      // For a non-placement new-expression, 'operator delete' can take a
      // size and/or an alignment if it has the right parameters.
      params = operatorDelete->getUsualDeleteParams();
    }

    assert(!params.DestroyingDelete &&
           "should not call destroying delete in a new-expression");

    // The second argument can be a std::size_t (for non-placement delete).
    if (params.Size)
      deleteArgs.add(Traits::get(cgf, allocSize),
                     cgf.getContext().getSizeType());

    // The next (second or third) argument can be a std::align_val_t, which
    // is an enum whose underlying type is std::size_t.
    // FIXME: Use the right type as the parameter type. Note that in a call
    // to operator delete(size_t, ...), we may not have it available.
    if (isAlignedAllocation(params.Alignment))
      cgf.cgm.errorNYI("CallDeleteDuringNew: aligned allocation");

    // Pass the rest of the arguments, which must match exactly.
    for (unsigned i = 0; i != numPlacementArgs; ++i) {
      auto arg = getPlacementArgs()[i];
      deleteArgs.add(Traits::get(cgf, arg.argValue), arg.argType);
    }

    // Call 'operator delete'.
    emitNewDeleteCall(cgf, operatorDelete, fpt, deleteArgs);
  }
};
} // namespace

/// Enter a cleanup to call 'operator delete' if the initializer in a
/// new-expression throws.
static void enterNewDeleteCleanup(CIRGenFunction &cgf, const CXXNewExpr *e,
                                  Address newPtr, mlir::Value allocSize,
                                  CharUnits allocAlign,
                                  const CallArgList &newArgs) {
  unsigned numNonPlacementArgs = e->getNumImplicitArgs();

  // If we're not inside a conditional branch, then the cleanup will
  // dominate and we can do the easier (and more efficient) thing.
  if (!cgf.isInConditionalBranch()) {
    struct DirectCleanupTraits {
      typedef mlir::Value ValueTy;
      typedef RValue RValueTy;
      static RValue get(CIRGenFunction &, ValueTy v) { return RValue::get(v); }
      static RValue get(CIRGenFunction &, RValueTy v) { return v; }
    };

    typedef CallDeleteDuringNew<DirectCleanupTraits> DirectCleanup;

    assert(!cir::MissingFeatures::typeAwareAllocation());
    cgf.ehStack.pushCleanupWithExtra<DirectCleanup>(
        EHCleanup, e->getNumPlacementArgs(), e->getOperatorDelete(),
        newPtr.getPointer(), allocSize, e->implicitAllocationParameters(),
        allocAlign, &newArgs, numNonPlacementArgs, &cgf,
        cgf.getLoc(e->getSourceRange()));

    return;
  }

  cgf.cgm.errorNYI(e->getSourceRange(),
                   "enterNewDeleteCleanup: conditional branch");
}

static void storeAnyExprIntoOneUnit(CIRGenFunction &cgf, const Expr *init,
                                    QualType allocType, Address newPtr,
                                    AggValueSlot::Overlap_t mayOverlap) {
  // FIXME: Refactor with emitExprAsInit.
  switch (cgf.getEvaluationKind(allocType)) {
  case cir::TEK_Scalar:
    cgf.emitScalarInit(init, cgf.getLoc(init->getSourceRange()),
                       cgf.makeAddrLValue(newPtr, allocType), false);
    return;
  case cir::TEK_Complex:
    cgf.emitComplexExprIntoLValue(init, cgf.makeAddrLValue(newPtr, allocType),
                                  /*isInit*/ true);
    return;
  case cir::TEK_Aggregate: {
    assert(!cir::MissingFeatures::aggValueSlotGC());
    assert(!cir::MissingFeatures::sanitizers());
    AggValueSlot slot = AggValueSlot::forAddr(
        newPtr, allocType.getQualifiers(), AggValueSlot::IsDestructed,
        AggValueSlot::IsNotAliased, mayOverlap, AggValueSlot::IsNotZeroed);
    cgf.emitAggExpr(init, slot);
    return;
  }
  }
  llvm_unreachable("bad evaluation kind");
}

void CIRGenFunction::emitNewArrayInitializer(
    const CXXNewExpr *e, QualType elementType, mlir::Type elementTy,
    Address beginPtr, mlir::Value numElements,
    mlir::Value allocSizeWithoutCookie) {
  // If we have a type with trivial initialization and no initializer,
  // there's nothing to do.
  if (!e->hasInitializer())
    return;

  Address curPtr = beginPtr;

  unsigned initListElements = 0;

  const Expr *init = e->getInitializer();
  Address endOfInit = Address::invalid();
  QualType::DestructionKind dtorKind = elementType.isDestructedType();
  assert(!cir::MissingFeatures::cleanupDeactivationScope());

  // Attempt to perform zero-initialization using memset.
  auto tryMemsetInitialization = [&]() -> bool {
    mlir::Location loc = numElements.getLoc();

    // FIXME: If the type is a pointer-to-data-member under the Itanium ABI,
    // we can initialize with a memset to -1.
    if (!cgm.getTypes().isZeroInitializable(elementType))
      return false;

    // Optimization: since zero initialization will just set the memory
    // to all zeroes, generate a single memset to do it in one shot.

    // Subtract out the size of any elements we've already initialized.
    auto remainingSize = allocSizeWithoutCookie;
    if (initListElements) {
      // We know this can't overflow; we check this when doing the allocation.
      unsigned initializedSize =
          getContext().getTypeSizeInChars(elementType).getQuantity() *
          initListElements;
      cir::ConstantOp initSizeOp =
          builder.getConstInt(loc, remainingSize.getType(), initializedSize);
      remainingSize = builder.createSub(loc, remainingSize, initSizeOp);
    }

    // Create the memset.
    mlir::Value castOp =
        builder.createPtrBitcast(curPtr.getPointer(), cgm.voidTy);
    builder.createMemSet(loc, castOp, builder.getConstInt(loc, cgm.uInt8Ty, 0),
                         remainingSize);
    return true;
  };

  const InitListExpr *ile = dyn_cast<InitListExpr>(init);
  const CXXParenListInitExpr *cplie = nullptr;
  const StringLiteral *sl = nullptr;
  const ObjCEncodeExpr *ocee = nullptr;
  const Expr *ignoreParen = nullptr;
  if (!ile) {
    ignoreParen = init->IgnoreParenImpCasts();
    cplie = dyn_cast<CXXParenListInitExpr>(ignoreParen);
    sl = dyn_cast<StringLiteral>(ignoreParen);
    ocee = dyn_cast<ObjCEncodeExpr>(ignoreParen);
  }
  // If the initializer is an initializer list, first do the explicit elements.
  if (ile || cplie || sl || ocee) {
    // Initializing from a (braced) string literal is a special case; the init
    // list element does not initialize a (single) array element.
    if ((ile && ile->isStringLiteralInit()) || sl || ocee) {
      cgm.errorNYI(ile->getSourceRange(),
                   "emitNewArrayInitializer: string literal init");
      return;
    }

    ArrayRef<const Expr *> initExprs =
        ile ? ile->inits() : cplie->getInitExprs();
    initListElements = initExprs.size();

    // If this is a multi-dimensional array new, we will initialize multiple
    // elements with each init list element.
    QualType allocType = e->getAllocatedType();
    if (const ConstantArrayType *cat = dyn_cast_or_null<ConstantArrayType>(
            allocType->getAsArrayTypeUnsafe())) {
      (void)cat;
      cgm.errorNYI(ile->getSourceRange(),
                   "emitNewArrayInitializer: constant array init");
      return;
    }

    // Enter a partial-destruction Cleanup if necessary.
    if (dtorKind) {
      cgm.errorNYI(ile->getSourceRange(),
                   "emitNewArrayInitializer: init requires dtor");
      return;
    }

    CharUnits elementSize = getContext().getTypeSizeInChars(elementType);
    CharUnits startAlign = curPtr.getAlignment();
    unsigned i = 0;
    for (const Expr *ie : initExprs) {
      // Tell the cleanup that it needs to destroy up to this
      // element.  TODO: some of these stores can be trivially
      // observed to be unnecessary.
      if (endOfInit.isValid()) {
        cgm.errorNYI(ie->getSourceRange(),
                     "emitNewArrayInitializer: update dtor cleanup ptr");
        return;
      }
      // FIXME: If the last initializer is an incomplete initializer list for
      // an array, and we have an array filler, we can fold together the two
      // initialization loops.
      storeAnyExprIntoOneUnit(*this, ie, ie->getType(), curPtr,
                              AggValueSlot::DoesNotOverlap);
      mlir::Location loc = getLoc(ie->getExprLoc());
      mlir::Value castOp = builder.createPtrBitcast(
          curPtr.getPointer(), convertTypeForMem(allocType));
      mlir::Value offsetOp = builder.getSignedInt(loc, 1, /*width=*/32);
      mlir::Value dataPtr = builder.createPtrStride(loc, castOp, offsetOp);
      curPtr = Address(dataPtr, curPtr.getElementType(),
                       startAlign.alignmentAtOffset((++i) * elementSize));
    }

    // The remaining elements are filled with the array filler expression.
    init = ile ? ile->getArrayFiller() : cplie->getArrayFiller();

    // Extract the initializer for the individual array elements by pulling
    // out the array filler from all the nested initializer lists. This avoids
    // generating a nested loop for the initialization.
    while (init && init->getType()->isConstantArrayType()) {
      auto *subIle = dyn_cast<InitListExpr>(init);
      if (!subIle)
        break;
      assert(subIle->getNumInits() == 0 && "explicit inits in array filler?");
      init = subIle->getArrayFiller();
    }

    // Switch back to initializing one base element at a time.
    curPtr = curPtr.withElementType(builder, beginPtr.getElementType());
  }

  // If all elements have already been initialized, skip any further
  // initialization.
  auto constOp = mlir::dyn_cast<cir::ConstantOp>(numElements.getDefiningOp());
  if (constOp) {
    auto constIntAttr = mlir::dyn_cast<cir::IntAttr>(constOp.getValue());
    // Just skip out if the constant count is zero.
    if (constIntAttr && constIntAttr.getUInt() <= initListElements)
      return;
  }

  assert(init && "have trailing elements to initialize but no initializer");

  // If this is a constructor call, try to optimize it out, and failing that
  // emit a single loop to initialize all remaining elements.
  if (const CXXConstructExpr *cce = dyn_cast<CXXConstructExpr>(init)) {
    CXXConstructorDecl *ctor = cce->getConstructor();
    if (ctor->isTrivial()) {
      // If new expression did not specify value-initialization, then there
      // is no initialization.
      if (!cce->requiresZeroInitialization())
        return;

      cgm.errorNYI(cce->getSourceRange(),
                   "emitNewArrayInitializer: trivial ctor zero-init");
      return;
    }

    // Store the new Cleanup position for irregular Cleanups.
    //
    // FIXME: Share this cleanup with the constructor call emission rather than
    // having it create a cleanup of its own.
    if (endOfInit.isValid())
      builder.createStore(getLoc(e->getSourceRange()), curPtr.emitRawPointer(),
                          endOfInit);

    mlir::Type initType = convertType(cce->getType());
    // Emit a constructor call loop to initialize the remaining elements.
    if (initListElements) {
      // If the number of elements is a constant, we will have already gotten
      // the constant op above. Here we use it to get the number of remaining
      // elements as a new constant.
      if (constOp) {
        auto constIntAttr = mlir::cast<cir::IntAttr>(constOp.getValue());
        uint64_t numRemainingElements =
            constIntAttr.getUInt() - initListElements;
        numElements =
            builder.getConstInt(getLoc(e->getSourceRange()),
                                numElements.getType(), numRemainingElements);
        // Currently, the AST gives us a pointer to the element type here
        // rather than an array. That's inconsistent with what it does
        // without an explicit initializer list, so we need to create an
        // array type here. That will decay back to a pointer when we lower
        // the cir.array.ctor op, but we need an array type for the initial
        // representation.
        if (!mlir::isa<cir::ArrayType>(initType))
          initType = cir::ArrayType::get(initType, numRemainingElements);
      } else {
        cgm.errorNYI(e->getSourceRange(),
                     "emitNewArrayInitializer: numRemainingElements with "
                     "non-constant count");
        return;
      }
    }

    curPtr = curPtr.withElementType(builder, initType);
    emitCXXAggrConstructorCall(ctor, numElements, curPtr, cce,
                               /*newPointerIsChecked=*/true,
                               cce->requiresZeroInitialization());
    if (getContext().getTargetInfo().emitVectorDeletingDtors(
            getContext().getLangOpts())) {
      cgm.errorNYI(e->getSourceRange(),
                   "emitNewArrayInitializer: emitVectorDeletingDtors");
    }
    return;
  }

  // If this is value-initialization, we can usually use memset.
  if (isa<ImplicitValueInitExpr>(init)) {
    if (tryMemsetInitialization())
      return;
    cgm.errorNYI(init->getSourceRange(),
                 "emitNewArrayInitializer: implicit value init");
    return;
  }

  cgm.errorNYI(init->getSourceRange(),
               "emitNewArrayInitializer: unsupported initializer");
  return;
}

static void emitNewInitializer(CIRGenFunction &cgf, const CXXNewExpr *e,
                               QualType elementType, mlir::Type elementTy,
                               Address newPtr, mlir::Value numElements,
                               mlir::Value allocSizeWithoutCookie) {
  assert(!cir::MissingFeatures::generateDebugInfo());
  if (e->isArray()) {
    cgf.emitNewArrayInitializer(e, elementType, elementTy, newPtr, numElements,
                                allocSizeWithoutCookie);
  } else if (const Expr *init = e->getInitializer()) {
    storeAnyExprIntoOneUnit(cgf, init, e->getAllocatedType(), newPtr,
                            AggValueSlot::DoesNotOverlap);
  }
}

RValue CIRGenFunction::emitCXXDestructorCall(
    GlobalDecl dtor, const CIRGenCallee &callee, mlir::Value thisVal,
    QualType thisTy, mlir::Value implicitParam, QualType implicitParamTy,
    const CallExpr *ce) {
  const CXXMethodDecl *dtorDecl = cast<CXXMethodDecl>(dtor.getDecl());

  assert(!thisTy.isNull());
  assert(thisTy->getAsCXXRecordDecl() == dtorDecl->getParent() &&
         "Pointer/Object mixup");

  assert(!cir::MissingFeatures::addressSpace());

  CallArgList args;
  commonBuildCXXMemberOrOperatorCall(*this, dtorDecl, thisVal, implicitParam,
                                     implicitParamTy, ce, args, nullptr);
  assert((ce || dtor.getDecl()) && "expected source location provider");
  assert(!cir::MissingFeatures::opCallMustTail());
  return emitCall(cgm.getTypes().arrangeCXXStructorDeclaration(dtor), callee,
                  ReturnValueSlot(), args, nullptr,
                  ce ? getLoc(ce->getExprLoc())
                     : getLoc(dtor.getDecl()->getSourceRange()));
}

RValue CIRGenFunction::emitCXXPseudoDestructorExpr(
    const CXXPseudoDestructorExpr *expr) {
  QualType destroyedType = expr->getDestroyedType();
  if (destroyedType.hasStrongOrWeakObjCLifetime()) {
    assert(!cir::MissingFeatures::objCLifetime());
    cgm.errorNYI(expr->getExprLoc(),
                 "emitCXXPseudoDestructorExpr: Objective-C lifetime is NYI");
  } else {
    // C++ [expr.pseudo]p1:
    //   The result shall only be used as the operand for the function call
    //   operator (), and the result of such a call has type void. The only
    //   effect is the evaluation of the postfix-expression before the dot or
    //   arrow.
    emitIgnoredExpr(expr->getBase());
  }

  return RValue::get(nullptr);
}

namespace {
/// Calls the given 'operator delete' on a single object.
struct CallObjectDelete final : EHScopeStack::Cleanup {
  mlir::Value ptr;
  const FunctionDecl *operatorDelete;
  QualType elementType;

  CallObjectDelete(mlir::Value ptr, const FunctionDecl *operatorDelete,
                   QualType elementType)
      : ptr(ptr), operatorDelete(operatorDelete), elementType(elementType) {}

  void emit(CIRGenFunction &cgf, Flags flags) override {
    cgf.emitDeleteCall(operatorDelete, ptr, elementType);
  }
};
} // namespace

/// Emit the code for deleting a single object.
static void emitObjectDelete(CIRGenFunction &cgf, const CXXDeleteExpr *de,
                             Address ptr, QualType elementType) {
  // C++11 [expr.delete]p3:
  //   If the static type of the object to be deleted is different from its
  //   dynamic type, the static type shall be a base class of the dynamic type
  //   of the object to be deleted and the static type shall have a virtual
  //   destructor or the behavior is undefined.
  assert(!cir::MissingFeatures::emitTypeCheck());

  const FunctionDecl *operatorDelete = de->getOperatorDelete();
  assert(!operatorDelete->isDestroyingOperatorDelete());

  // Find the destructor for the type, if applicable.  If the
  // destructor is virtual, we'll just emit the vcall and return.
  const CXXDestructorDecl *dtor = nullptr;
  if (const auto *rd = elementType->getAsCXXRecordDecl()) {
    if (rd->hasDefinition() && !rd->hasTrivialDestructor()) {
      dtor = rd->getDestructor();

      if (dtor->isVirtual()) {
        assert(!cir::MissingFeatures::devirtualizeDestructor());
        cgf.cgm.getCXXABI().emitVirtualObjectDelete(cgf, de, ptr, elementType,
                                                    dtor);
        return;
      }
    }
  }

  // Make sure that we call delete even if the dtor throws.
  // This doesn't have to a conditional cleanup because we're going
  // to pop it off in a second.
  cgf.ehStack.pushCleanup<CallObjectDelete>(
      NormalAndEHCleanup, ptr.getPointer(), operatorDelete, elementType);

  if (dtor) {
    cgf.emitCXXDestructorCall(dtor, Dtor_Complete,
                              /*ForVirtualBase=*/false,
                              /*Delegating=*/false, ptr, elementType);
  } else if (elementType.getObjCLifetime()) {
    assert(!cir::MissingFeatures::objCLifetime());
    cgf.cgm.errorNYI(de->getSourceRange(), "emitObjectDelete: ObjCLifetime");
  }

  cgf.popCleanupBlock();
}

void CIRGenFunction::emitCXXDeleteExpr(const CXXDeleteExpr *e) {
  const Expr *arg = e->getArgument();
  Address ptr = emitPointerWithAlignment(arg);

  // Null check the pointer.
  //
  // We could avoid this null check if we can determine that the object
  // destruction is trivial and doesn't require an array cookie; we can
  // unconditionally perform the operator delete call in that case. For now, we
  // assume that deleted pointers are null rarely enough that it's better to
  // keep the branch. This might be worth revisiting for a -O0 code size win.
  assert(!cir::MissingFeatures::emitNullCheckForDeleteCalls());
  cir::YieldOp thenYield;
  mlir::Value notNull = builder.createPtrIsNotNull(ptr.getPointer());
  cir::IfOp::create(builder, getLoc(e->getExprLoc()), notNull,
                    /*withElseRegion=*/false,
                    /*thenBuilder=*/
                    [&](mlir::OpBuilder &b, mlir::Location loc) {
                      thenYield = builder.createYield(loc);
                    });

  // Emit the rest of the CIR inside the if-op's then region, but restore the
  // insertion point to the point after the if when this function returns.
  mlir::OpBuilder::InsertionGuard guard(builder);
  builder.setInsertionPoint(thenYield);

  QualType deleteTy = e->getDestroyedType();

  // A destroying operator delete overrides the entire operation of the
  // delete expression.
  if (e->getOperatorDelete()->isDestroyingOperatorDelete()) {
    cgm.errorNYI(e->getSourceRange(),
                 "emitCXXDeleteExpr: destroying operator delete");
    return;
  }

  // We might be deleting a pointer to array.
  deleteTy = getContext().getBaseElementType(deleteTy);
  ptr = ptr.withElementType(builder, convertTypeForMem(deleteTy));

  if (e->isArrayForm() &&
      cgm.getASTContext().getTargetInfo().emitVectorDeletingDtors(
          cgm.getASTContext().getLangOpts())) {
    cgm.errorNYI(e->getSourceRange(),
                 "emitCXXDeleteExpr: emitVectorDeletingDtors");
  }

  if (e->isArrayForm()) {
    const FunctionDecl *operatorDelete = e->getOperatorDelete();
    cir::FuncOp operatorDeleteFn = cgm.getAddrOfFunction(operatorDelete);
    auto deleteFn =
        mlir::FlatSymbolRefAttr::get(operatorDeleteFn.getSymNameAttr());
    UsualDeleteParams udp = operatorDelete->getUsualDeleteParams();
    auto deleteParams = cir::UsualDeleteParamsAttr::get(
        builder.getContext(), udp.Size, isAlignedAllocation(udp.Alignment),
        isTypeAwareAllocation(udp.TypeAwareDelete), udp.DestroyingDelete);

    mlir::FlatSymbolRefAttr elementDtor;
    if (const auto *rd = deleteTy->getAsCXXRecordDecl()) {
      if (rd->hasDefinition() && !rd->hasTrivialDestructor()) {
        const CXXDestructorDecl *dtor = rd->getDestructor();
        if (dtor->getType()->castAs<FunctionProtoType>()->canThrow())
          cgm.errorNYI(e->getSourceRange(),
                       "emitCXXDeleteExpr: throwing destructor");
        cir::FuncOp dtorFn =
            cgm.getAddrOfCXXStructor(GlobalDecl(dtor, Dtor_Complete));
        elementDtor = mlir::FlatSymbolRefAttr::get(builder.getContext(),
                                                   dtorFn.getSymNameAttr());
      }
    }

    cir::DeleteArrayOp::create(builder, ptr.getPointer().getLoc(),
                               ptr.getPointer(), deleteFn, deleteParams,
                               elementDtor);
  } else {
    emitObjectDelete(*this, e, ptr, deleteTy);
  }
}

mlir::Value CIRGenFunction::emitCXXNewExpr(const CXXNewExpr *e) {
  // The element type being allocated.
  QualType allocType = getContext().getBaseElementType(e->getAllocatedType());

  // 1. Build a call to the allocation function.
  FunctionDecl *allocator = e->getOperatorNew();

  // If there is a brace-initializer, cannot allocate fewer elements than inits.
  unsigned minElements = 0;
  if (e->isArray() && e->hasInitializer()) {
    const InitListExpr *ile = dyn_cast<InitListExpr>(e->getInitializer());
    if (ile && ile->isStringLiteralInit())
      minElements =
          cast<ConstantArrayType>(ile->getType()->getAsArrayTypeUnsafe())
              ->getSize()
              .getZExtValue();
    else if (ile)
      minElements = ile->getNumInits();
  }

  mlir::Value numElements = nullptr;
  mlir::Value allocSizeWithoutCookie = nullptr;
  mlir::Value allocSize = emitCXXNewAllocSize(
      *this, e, minElements, numElements, allocSizeWithoutCookie);
  CharUnits allocAlign = getContext().getTypeAlignInChars(allocType);

  // Emit the allocation call.
  Address allocation = Address::invalid();
  CallArgList allocatorArgs;
  if (allocator->isReservedGlobalPlacementOperator()) {
    // If the allocator is a global placement operator, just
    // "inline" it directly.
    assert(e->getNumPlacementArgs() == 1);
    const Expr *arg = *e->placement_arguments().begin();

    LValueBaseInfo baseInfo;
    allocation = emitPointerWithAlignment(arg, &baseInfo);

    // The pointer expression will, in many cases, be an opaque void*.
    // In these cases, discard the computed alignment and use the
    // formal alignment of the allocated type.
    if (baseInfo.getAlignmentSource() != AlignmentSource::Decl)
      allocation = allocation.withAlignment(allocAlign);

    // Set up allocatorArgs for the call to operator delete if it's not
    // the reserved global operator.
    if (e->getOperatorDelete() &&
        !e->getOperatorDelete()->isReservedGlobalPlacementOperator()) {
      cgm.errorNYI(e->getSourceRange(),
                   "emitCXXNewExpr: reserved placement new with delete");
    }
  } else {
    const FunctionProtoType *allocatorType =
        allocator->getType()->castAs<FunctionProtoType>();
    unsigned paramsToSkip = 0;

    // The allocation size is the first argument.
    QualType sizeType = getContext().getSizeType();
    allocatorArgs.add(RValue::get(allocSize), sizeType);
    ++paramsToSkip;

    if (allocSize != allocSizeWithoutCookie) {
      CharUnits cookieAlign = getSizeAlign(); // FIXME: Ask the ABI.
      allocAlign = std::max(allocAlign, cookieAlign);
    }

    // The allocation alignment may be passed as the second argument.
    if (e->passAlignment()) {
      cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: pass alignment");
    }

    // FIXME: Why do we not pass a CalleeDecl here?
    emitCallArgs(allocatorArgs, allocatorType, e->placement_arguments(),
                 AbstractCallee(), paramsToSkip);
    RValue rv =
        emitNewDeleteCall(*this, allocator, allocatorType, allocatorArgs);

    // Set !heapallocsite metadata on the call to operator new.
    assert(!cir::MissingFeatures::generateDebugInfo());

    // If this was a call to a global replaceable allocation function that does
    // not take an alignment argument, the allocator is known to produce storage
    // that's suitably aligned for any object that fits, up to a known
    // threshold. Otherwise assume it's suitably aligned for the allocated type.
    CharUnits allocationAlign = allocAlign;
    if (!e->passAlignment() &&
        allocator->isReplaceableGlobalAllocationFunction()) {
      const TargetInfo &target = cgm.getASTContext().getTargetInfo();
      unsigned allocatorAlign = llvm::bit_floor(std::min<uint64_t>(
          target.getNewAlign(), getContext().getTypeSize(allocType)));
      allocationAlign = std::max(
          allocationAlign, getContext().toCharUnitsFromBits(allocatorAlign));
    }

    mlir::Value allocPtr = rv.getValue();
    allocation = Address(
        allocPtr, mlir::cast<cir::PointerType>(allocPtr.getType()).getPointee(),
        allocationAlign);
  }

  // Emit a null check on the allocation result if the allocation
  // function is allowed to return null (because it has a non-throwing
  // exception spec or is the reserved placement new) and we have an
  // interesting initializer will be running sanitizers on the initialization.
  bool nullCheck = e->shouldNullCheckAllocation() &&
                   (!allocType.isPODType(getContext()) || e->hasInitializer());
  assert(!cir::MissingFeatures::exprNewNullCheck());
  if (nullCheck)
    cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: null check");

  // If there's an operator delete, enter a cleanup to call it if an
  // exception is thrown. If we do this, we'll be creating the result pointer
  // inside a cleanup scope, either with a bitcast or an offset based on the
  // array cookie size. However, we need to return that pointer from outside
  // the cleanup scope, so we need to store it in a temporary variable.
  bool useNewDeleteCleanup =
      e->getOperatorDelete() &&
      !e->getOperatorDelete()->isReservedGlobalPlacementOperator();
  EHScopeStack::stable_iterator operatorDeleteCleanup;
  mlir::Operation *cleanupDominator = nullptr;
  if (useNewDeleteCleanup) {
    assert(!cir::MissingFeatures::typeAwareAllocation());
    enterNewDeleteCleanup(*this, e, allocation, allocSize, allocAlign,
                          allocatorArgs);
    operatorDeleteCleanup = ehStack.stable_begin();
    cleanupDominator =
        cir::UnreachableOp::create(builder, getLoc(e->getSourceRange()))
            .getOperation();
  }

  if (allocSize != allocSizeWithoutCookie) {
    assert(e->isArray());
    allocation = cgm.getCXXABI().initializeArrayCookie(
        *this, allocation, numElements, e, allocType);
  }

  mlir::Type elementTy;
  if (e->isArray()) {
    // For array new, use the allocated type to handle multidimensional arrays
    // correctly
    elementTy = convertTypeForMem(e->getAllocatedType());
  } else {
    elementTy = convertTypeForMem(allocType);
  }
  Address result = builder.createElementBitCast(getLoc(e->getSourceRange()),
                                                allocation, elementTy);

  // If we're inside a new delete cleanup, store the result pointer.
  Address resultPtr = Address::invalid();
  if (useNewDeleteCleanup) {
    resultPtr =
        createTempAlloca(builder.getPointerTo(elementTy), result.getAlignment(),
                         getLoc(e->getSourceRange()), "__new_result");
    builder.createStore(getLoc(e->getSourceRange()), result.getPointer(),
                        resultPtr);
  }

  // Passing pointer through launder.invariant.group to avoid propagation of
  // vptrs information which may be included in previous type.
  // To not break LTO with different optimizations levels, we do it regardless
  // of optimization level.
  if (cgm.getCodeGenOpts().StrictVTablePointers &&
      allocator->isReservedGlobalPlacementOperator())
    cgm.errorNYI(e->getSourceRange(), "emitCXXNewExpr: strict vtable pointers");

  assert(!cir::MissingFeatures::sanitizers());

  emitNewInitializer(*this, e, allocType, elementTy, result, numElements,
                     allocSizeWithoutCookie);

  // Deactivate the 'operator delete' cleanup if we finished
  // initialization.
  if (useNewDeleteCleanup) {
    assert(operatorDeleteCleanup.isValid());
    assert(resultPtr.isValid());
    deactivateCleanupBlock(operatorDeleteCleanup, cleanupDominator);
    cleanupDominator->erase();
    cir::LoadOp loadResult =
        builder.createLoad(getLoc(e->getSourceRange()), resultPtr);
    result = result.withPointer(loadResult.getResult());
  }

  assert(!cir::MissingFeatures::exprNewNullCheck());

  return result.getPointer();
}

void CIRGenFunction::emitDeleteCall(const FunctionDecl *deleteFD,
                                    mlir::Value ptr, QualType deleteTy) {
  assert(!cir::MissingFeatures::deleteArray());

  const auto *deleteFTy = deleteFD->getType()->castAs<FunctionProtoType>();
  CallArgList deleteArgs;

  UsualDeleteParams params = deleteFD->getUsualDeleteParams();
  auto paramTypeIt = deleteFTy->param_type_begin();

  // Pass std::type_identity tag if present
  if (isTypeAwareAllocation(params.TypeAwareDelete))
    cgm.errorNYI(deleteFD->getSourceRange(),
                 "emitDeleteCall: type aware delete");

  // Pass the pointer itself.
  QualType argTy = *paramTypeIt++;
  mlir::Value deletePtr =
      builder.createBitcast(ptr.getLoc(), ptr, convertType(argTy));
  deleteArgs.add(RValue::get(deletePtr), argTy);

  // Pass the std::destroying_delete tag if present.
  if (params.DestroyingDelete)
    cgm.errorNYI(deleteFD->getSourceRange(),
                 "emitDeleteCall: destroying delete");

  // Pass the size if the delete function has a size_t parameter.
  if (params.Size) {
    QualType sizeType = *paramTypeIt++;
    CharUnits deleteTypeSize = getContext().getTypeSizeInChars(deleteTy);
    assert(mlir::isa<cir::IntType>(convertType(sizeType)) &&
           "expected cir::IntType");
    cir::ConstantOp size = builder.getConstInt(
        *currSrcLoc, convertType(sizeType), deleteTypeSize.getQuantity());

    deleteArgs.add(RValue::get(size), sizeType);
  }

  // Pass the alignment if the delete function has an align_val_t parameter.
  if (isAlignedAllocation(params.Alignment))
    cgm.errorNYI(deleteFD->getSourceRange(),
                 "emitDeleteCall: aligned allocation");

  assert(paramTypeIt == deleteFTy->param_type_end() &&
         "unknown parameter to usual delete function");

  // Emit the call to delete.
  emitNewDeleteCall(*this, deleteFD, deleteFTy, deleteArgs);
}

static mlir::Value emitDynamicCastToNull(CIRGenFunction &cgf,
                                         mlir::Location loc, QualType destTy) {
  mlir::Type destCIRTy = cgf.convertType(destTy);
  assert(mlir::isa<cir::PointerType>(destCIRTy) &&
         "result of dynamic_cast should be a ptr");

  if (!destTy->isPointerType()) {
    mlir::Region *currentRegion = cgf.getBuilder().getBlock()->getParent();
    /// C++ [expr.dynamic.cast]p9:
    ///   A failed cast to reference type throws std::bad_cast
    cgf.cgm.getCXXABI().emitBadCastCall(cgf, loc);

    // The call to bad_cast will terminate the current block. Create a new block
    // to hold any follow up code.
    cgf.getBuilder().createBlock(currentRegion, currentRegion->end());
  }

  return cgf.getBuilder().getNullPtr(destCIRTy, loc);
}

mlir::Value CIRGenFunction::emitDynamicCast(Address thisAddr,
                                            const CXXDynamicCastExpr *dce) {
  mlir::Location loc = getLoc(dce->getSourceRange());

  cgm.emitExplicitCastExprType(dce, this);
  QualType destTy = dce->getTypeAsWritten();
  QualType srcTy = dce->getSubExpr()->getType();

  // C++ [expr.dynamic.cast]p7:
  //   If T is "pointer to cv void," then the result is a pointer to the most
  //   derived object pointed to by v.
  bool isDynCastToVoid = destTy->isVoidPointerType();
  bool isRefCast = destTy->isReferenceType();

  QualType srcRecordTy;
  QualType destRecordTy;
  if (isDynCastToVoid) {
    srcRecordTy = srcTy->getPointeeType();
    // No destRecordTy.
  } else if (const PointerType *destPTy = destTy->getAs<PointerType>()) {
    srcRecordTy = srcTy->castAs<PointerType>()->getPointeeType();
    destRecordTy = destPTy->getPointeeType();
  } else {
    srcRecordTy = srcTy;
    destRecordTy = destTy->castAs<ReferenceType>()->getPointeeType();
  }

  assert(srcRecordTy->isRecordType() && "source type must be a record type!");
  assert(!cir::MissingFeatures::emitTypeCheck());

  if (dce->isAlwaysNull())
    return emitDynamicCastToNull(*this, loc, destTy);

  auto destCirTy = mlir::cast<cir::PointerType>(convertType(destTy));
  return cgm.getCXXABI().emitDynamicCast(*this, loc, srcRecordTy, destRecordTy,
                                         destCirTy, isRefCast, thisAddr);
}

static mlir::Value emitCXXTypeidFromVTable(CIRGenFunction &cgf, const Expr *e,
                                           mlir::Type typeInfoPtrTy,
                                           bool hasNullCheck) {
  Address thisPtr = cgf.emitLValue(e).getAddress();
  QualType srcType = e->getType();

  // C++ [class.cdtor]p4:
  //   If the operand of typeid refers to the object under construction or
  //   destruction and the static type of the operand is neither the constructor
  //   or destructor’s class nor one of its bases, the behavior is undefined.
  assert(!cir::MissingFeatures::sanitizers());

  if (hasNullCheck && cgf.cgm.getCXXABI().shouldTypeidBeNullChecked(srcType)) {
    mlir::Value isThisNull =
        cgf.getBuilder().createPtrIsNull(thisPtr.getPointer());
    // We don't really care about the value, we just want to make sure the
    // 'true' side calls bad-type-id.
    cir::IfOp::create(
        cgf.getBuilder(), cgf.getLoc(e->getSourceRange()), isThisNull,
        /*withElseRegion=*/false, [&](mlir::OpBuilder &, mlir::Location loc) {
          cgf.cgm.getCXXABI().emitBadTypeidCall(cgf, loc);
        });
  }

  return cgf.cgm.getCXXABI().emitTypeid(cgf, srcType, thisPtr, typeInfoPtrTy);
}

mlir::Value CIRGenFunction::emitCXXTypeidExpr(const CXXTypeidExpr *e) {
  mlir::Location loc = getLoc(e->getSourceRange());
  mlir::Type resultType = cir::PointerType::get(convertType(e->getType()));
  QualType ty = e->isTypeOperand() ? e->getTypeOperand(getContext())
                                   : e->getExprOperand()->getType();

  // If the non-default global var address space is not default, we need to do
  // an address-space cast here.
  assert(!cir::MissingFeatures::addressSpace());

  // C++ [expr.typeid]p2:
  //   When typeid is applied to a glvalue expression whose type is a
  //   polymorphic class type, the result refers to a std::type_info object
  //   representing the type of the most derived object (that is, the dynamic
  //   type) to which the glvalue refers.
  // If the operand is already most derived object, no need to look up vtable.
  if (!e->isTypeOperand() && e->isPotentiallyEvaluated() &&
      !e->isMostDerived(getContext()))
    return emitCXXTypeidFromVTable(*this, e->getExprOperand(), resultType,
                                   e->hasNullCheck());

  auto typeInfo =
      cast<cir::GlobalViewAttr>(cgm.getAddrOfRTTIDescriptor(loc, ty));
  // `getAddrOfRTTIDescriptor` lies to us and always gives us a uint8ptr as its
  // type, however we need the value of the actual global to call the
  // get-global-op, so look it up here.
  auto typeInfoGlobal =
      cast<cir::GlobalOp>(cgm.getGlobalValue(typeInfo.getSymbol().getValue()));
  auto getTypeInfo = cir::GetGlobalOp::create(
      builder, loc, builder.getPointerTo(typeInfoGlobal.getSymType()),
      typeInfoGlobal.getSymName());
  // The ABI is just generating these sometimes as ptr to u8, but they are
  // simply a representation of the type_info. So we have to cast this, if
  // necessary (createBitcast is a noop if the types match).
  return builder.createBitcast(getTypeInfo, resultType);
}