d1592a966b
When attempting to replace a full vector constant load with an instruction that uses a smaller constant, check the scheduler model to ensure the instruction isn't slower. Throughput must not regress, but allow a small increase in latency based on how much constant data we're saving (I've used a simple estimate of 1 cycle per 128-bits of data saved). NOTE: this currently ignores hoisted constant loads where the slower instruction might be acceptable. Fixes #135998
796 lines
32 KiB
C++
796 lines
32 KiB
C++
//===-- X86FixupVectorConstants.cpp - optimize constant generation -------===//
|
|
//
|
|
// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
|
|
// See https://llvm.org/LICENSE.txt for license information.
|
|
// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
//
|
|
// This file examines all full size vector constant pool loads and attempts to
|
|
// replace them with smaller constant pool entries, including:
|
|
// * Converting AVX512 memory-fold instructions to their broadcast-fold form.
|
|
// * Using vzload scalar loads.
|
|
// * Broadcasting of full width loads.
|
|
// * Sign/Zero extension of full width loads.
|
|
//
|
|
//===----------------------------------------------------------------------===//
|
|
|
|
#include "X86.h"
|
|
#include "X86InstrFoldTables.h"
|
|
#include "X86InstrInfo.h"
|
|
#include "X86Subtarget.h"
|
|
#include "llvm/ADT/Statistic.h"
|
|
#include "llvm/CodeGen/MachineConstantPool.h"
|
|
|
|
using namespace llvm;
|
|
|
|
#define DEBUG_TYPE "x86-fixup-vector-constants"
|
|
|
|
STATISTIC(NumInstChanges, "Number of instructions changes");
|
|
|
|
namespace {
|
|
class X86FixupVectorConstantsPass : public MachineFunctionPass {
|
|
public:
|
|
static char ID;
|
|
|
|
X86FixupVectorConstantsPass() : MachineFunctionPass(ID) {}
|
|
|
|
StringRef getPassName() const override {
|
|
return "X86 Fixup Vector Constants";
|
|
}
|
|
|
|
bool runOnMachineFunction(MachineFunction &MF) override;
|
|
bool processInstruction(MachineFunction &MF, MachineBasicBlock &MBB,
|
|
MachineInstr &MI);
|
|
|
|
// This pass runs after regalloc and doesn't support VReg operands.
|
|
MachineFunctionProperties getRequiredProperties() const override {
|
|
return MachineFunctionProperties().setNoVRegs();
|
|
}
|
|
|
|
private:
|
|
const X86InstrInfo *TII = nullptr;
|
|
const X86Subtarget *ST = nullptr;
|
|
const MCSchedModel *SM = nullptr;
|
|
};
|
|
} // end anonymous namespace
|
|
|
|
char X86FixupVectorConstantsPass::ID = 0;
|
|
|
|
INITIALIZE_PASS(X86FixupVectorConstantsPass, DEBUG_TYPE, DEBUG_TYPE, false, false)
|
|
|
|
FunctionPass *llvm::createX86FixupVectorConstants() {
|
|
return new X86FixupVectorConstantsPass();
|
|
}
|
|
|
|
/// Normally, we only allow poison in vector splats. However, as this is part
|
|
/// of the backend, and working with the DAG representation, which currently
|
|
/// only natively represents undef values, we need to accept undefs here.
|
|
static Constant *getSplatValueAllowUndef(const ConstantVector *C) {
|
|
Constant *Res = nullptr;
|
|
for (Value *Op : C->operands()) {
|
|
Constant *OpC = cast<Constant>(Op);
|
|
if (isa<UndefValue>(OpC))
|
|
continue;
|
|
if (!Res)
|
|
Res = OpC;
|
|
else if (Res != OpC)
|
|
return nullptr;
|
|
}
|
|
return Res;
|
|
}
|
|
|
|
// Attempt to extract the full width of bits data from the constant.
|
|
static std::optional<APInt> extractConstantBits(const Constant *C) {
|
|
unsigned NumBits = C->getType()->getPrimitiveSizeInBits();
|
|
|
|
if (isa<UndefValue>(C))
|
|
return APInt::getZero(NumBits);
|
|
|
|
if (auto *CInt = dyn_cast<ConstantInt>(C)) {
|
|
if (isa<VectorType>(CInt->getType()))
|
|
return APInt::getSplat(NumBits, CInt->getValue());
|
|
|
|
return CInt->getValue();
|
|
}
|
|
|
|
if (auto *CFP = dyn_cast<ConstantFP>(C)) {
|
|
if (isa<VectorType>(CFP->getType()))
|
|
return APInt::getSplat(NumBits, CFP->getValue().bitcastToAPInt());
|
|
|
|
return CFP->getValue().bitcastToAPInt();
|
|
}
|
|
|
|
if (auto *CV = dyn_cast<ConstantVector>(C)) {
|
|
if (auto *CVSplat = getSplatValueAllowUndef(CV)) {
|
|
if (std::optional<APInt> Bits = extractConstantBits(CVSplat)) {
|
|
assert((NumBits % Bits->getBitWidth()) == 0 && "Illegal splat");
|
|
return APInt::getSplat(NumBits, *Bits);
|
|
}
|
|
}
|
|
|
|
APInt Bits = APInt::getZero(NumBits);
|
|
for (unsigned I = 0, E = CV->getNumOperands(); I != E; ++I) {
|
|
Constant *Elt = CV->getOperand(I);
|
|
std::optional<APInt> SubBits = extractConstantBits(Elt);
|
|
if (!SubBits)
|
|
return std::nullopt;
|
|
assert(NumBits == (E * SubBits->getBitWidth()) &&
|
|
"Illegal vector element size");
|
|
Bits.insertBits(*SubBits, I * SubBits->getBitWidth());
|
|
}
|
|
return Bits;
|
|
}
|
|
|
|
if (auto *CDS = dyn_cast<ConstantDataSequential>(C)) {
|
|
bool IsInteger = CDS->getElementType()->isIntegerTy();
|
|
bool IsFloat = CDS->getElementType()->isHalfTy() ||
|
|
CDS->getElementType()->isBFloatTy() ||
|
|
CDS->getElementType()->isFloatTy() ||
|
|
CDS->getElementType()->isDoubleTy();
|
|
if (IsInteger || IsFloat) {
|
|
APInt Bits = APInt::getZero(NumBits);
|
|
unsigned EltBits = CDS->getElementType()->getPrimitiveSizeInBits();
|
|
for (unsigned I = 0, E = CDS->getNumElements(); I != E; ++I) {
|
|
if (IsInteger)
|
|
Bits.insertBits(CDS->getElementAsAPInt(I), I * EltBits);
|
|
else
|
|
Bits.insertBits(CDS->getElementAsAPFloat(I).bitcastToAPInt(),
|
|
I * EltBits);
|
|
}
|
|
return Bits;
|
|
}
|
|
}
|
|
|
|
return std::nullopt;
|
|
}
|
|
|
|
static std::optional<APInt> extractConstantBits(const Constant *C,
|
|
unsigned NumBits) {
|
|
if (std::optional<APInt> Bits = extractConstantBits(C))
|
|
return Bits->zextOrTrunc(NumBits);
|
|
return std::nullopt;
|
|
}
|
|
|
|
// Attempt to compute the splat width of bits data by normalizing the splat to
|
|
// remove undefs.
|
|
static std::optional<APInt> getSplatableConstant(const Constant *C,
|
|
unsigned SplatBitWidth) {
|
|
const Type *Ty = C->getType();
|
|
assert((Ty->getPrimitiveSizeInBits() % SplatBitWidth) == 0 &&
|
|
"Illegal splat width");
|
|
|
|
if (std::optional<APInt> Bits = extractConstantBits(C))
|
|
if (Bits->isSplat(SplatBitWidth))
|
|
return Bits->trunc(SplatBitWidth);
|
|
|
|
// Detect general splats with undefs.
|
|
// TODO: Do we need to handle NumEltsBits > SplatBitWidth splitting?
|
|
if (auto *CV = dyn_cast<ConstantVector>(C)) {
|
|
unsigned NumOps = CV->getNumOperands();
|
|
unsigned NumEltsBits = Ty->getScalarSizeInBits();
|
|
unsigned NumScaleOps = SplatBitWidth / NumEltsBits;
|
|
if ((SplatBitWidth % NumEltsBits) == 0) {
|
|
// Collect the elements and ensure that within the repeated splat sequence
|
|
// they either match or are undef.
|
|
SmallVector<Constant *, 16> Sequence(NumScaleOps, nullptr);
|
|
for (unsigned Idx = 0; Idx != NumOps; ++Idx) {
|
|
if (Constant *Elt = CV->getAggregateElement(Idx)) {
|
|
if (isa<UndefValue>(Elt))
|
|
continue;
|
|
unsigned SplatIdx = Idx % NumScaleOps;
|
|
if (!Sequence[SplatIdx] || Sequence[SplatIdx] == Elt) {
|
|
Sequence[SplatIdx] = Elt;
|
|
continue;
|
|
}
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
// Extract the constant bits forming the splat and insert into the bits
|
|
// data, leave undef as zero.
|
|
APInt SplatBits = APInt::getZero(SplatBitWidth);
|
|
for (unsigned I = 0; I != NumScaleOps; ++I) {
|
|
if (!Sequence[I])
|
|
continue;
|
|
if (std::optional<APInt> Bits = extractConstantBits(Sequence[I])) {
|
|
SplatBits.insertBits(*Bits, I * Bits->getBitWidth());
|
|
continue;
|
|
}
|
|
return std::nullopt;
|
|
}
|
|
return SplatBits;
|
|
}
|
|
}
|
|
|
|
return std::nullopt;
|
|
}
|
|
|
|
// Split raw bits into a constant vector of elements of a specific bit width.
|
|
// NOTE: We don't always bother converting to scalars if the vector length is 1.
|
|
static Constant *rebuildConstant(LLVMContext &Ctx, Type *SclTy,
|
|
const APInt &Bits, unsigned NumSclBits) {
|
|
unsigned BitWidth = Bits.getBitWidth();
|
|
|
|
if (NumSclBits == 8) {
|
|
SmallVector<uint8_t> RawBits;
|
|
for (unsigned I = 0; I != BitWidth; I += 8)
|
|
RawBits.push_back(Bits.extractBits(8, I).getZExtValue());
|
|
return ConstantDataVector::get(Ctx, RawBits);
|
|
}
|
|
|
|
if (NumSclBits == 16) {
|
|
SmallVector<uint16_t> RawBits;
|
|
for (unsigned I = 0; I != BitWidth; I += 16)
|
|
RawBits.push_back(Bits.extractBits(16, I).getZExtValue());
|
|
if (SclTy->is16bitFPTy())
|
|
return ConstantDataVector::getFP(SclTy, RawBits);
|
|
return ConstantDataVector::get(Ctx, RawBits);
|
|
}
|
|
|
|
if (NumSclBits == 32) {
|
|
SmallVector<uint32_t> RawBits;
|
|
for (unsigned I = 0; I != BitWidth; I += 32)
|
|
RawBits.push_back(Bits.extractBits(32, I).getZExtValue());
|
|
if (SclTy->isFloatTy())
|
|
return ConstantDataVector::getFP(SclTy, RawBits);
|
|
return ConstantDataVector::get(Ctx, RawBits);
|
|
}
|
|
|
|
assert(NumSclBits == 64 && "Unhandled vector element width");
|
|
|
|
SmallVector<uint64_t> RawBits;
|
|
for (unsigned I = 0; I != BitWidth; I += 64)
|
|
RawBits.push_back(Bits.extractBits(64, I).getZExtValue());
|
|
if (SclTy->isDoubleTy())
|
|
return ConstantDataVector::getFP(SclTy, RawBits);
|
|
return ConstantDataVector::get(Ctx, RawBits);
|
|
}
|
|
|
|
// Attempt to rebuild a normalized splat vector constant of the requested splat
|
|
// width, built up of potentially smaller scalar values.
|
|
static Constant *rebuildSplatCst(const Constant *C, unsigned /*NumBits*/,
|
|
unsigned /*NumElts*/, unsigned SplatBitWidth) {
|
|
// TODO: Truncate to NumBits once ConvertToBroadcastAVX512 support this.
|
|
std::optional<APInt> Splat = getSplatableConstant(C, SplatBitWidth);
|
|
if (!Splat)
|
|
return nullptr;
|
|
|
|
// Determine scalar size to use for the constant splat vector, clamping as we
|
|
// might have found a splat smaller than the original constant data.
|
|
Type *SclTy = C->getType()->getScalarType();
|
|
unsigned NumSclBits = SclTy->getPrimitiveSizeInBits();
|
|
NumSclBits = std::min<unsigned>(NumSclBits, SplatBitWidth);
|
|
|
|
// Fallback to i64 / double.
|
|
NumSclBits = (NumSclBits == 8 || NumSclBits == 16 || NumSclBits == 32)
|
|
? NumSclBits
|
|
: 64;
|
|
|
|
// Extract per-element bits.
|
|
return rebuildConstant(C->getContext(), SclTy, *Splat, NumSclBits);
|
|
}
|
|
|
|
static Constant *rebuildZeroUpperCst(const Constant *C, unsigned NumBits,
|
|
unsigned /*NumElts*/,
|
|
unsigned ScalarBitWidth) {
|
|
Type *SclTy = C->getType()->getScalarType();
|
|
unsigned NumSclBits = SclTy->getPrimitiveSizeInBits();
|
|
LLVMContext &Ctx = C->getContext();
|
|
|
|
if (NumBits > ScalarBitWidth) {
|
|
// Determine if the upper bits are all zero.
|
|
if (std::optional<APInt> Bits = extractConstantBits(C, NumBits)) {
|
|
if (Bits->countLeadingZeros() >= (NumBits - ScalarBitWidth)) {
|
|
// If the original constant was made of smaller elements, try to retain
|
|
// those types.
|
|
if (ScalarBitWidth > NumSclBits && (ScalarBitWidth % NumSclBits) == 0)
|
|
return rebuildConstant(Ctx, SclTy, *Bits, NumSclBits);
|
|
|
|
// Fallback to raw integer bits.
|
|
APInt RawBits = Bits->zextOrTrunc(ScalarBitWidth);
|
|
return ConstantInt::get(Ctx, RawBits);
|
|
}
|
|
}
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
|
|
static Constant *rebuildExtCst(const Constant *C, bool IsSExt,
|
|
unsigned NumBits, unsigned NumElts,
|
|
unsigned SrcEltBitWidth) {
|
|
unsigned DstEltBitWidth = NumBits / NumElts;
|
|
assert((NumBits % NumElts) == 0 && (NumBits % SrcEltBitWidth) == 0 &&
|
|
(DstEltBitWidth % SrcEltBitWidth) == 0 &&
|
|
(DstEltBitWidth > SrcEltBitWidth) && "Illegal extension width");
|
|
|
|
if (std::optional<APInt> Bits = extractConstantBits(C, NumBits)) {
|
|
assert((Bits->getBitWidth() / DstEltBitWidth) == NumElts &&
|
|
(Bits->getBitWidth() % DstEltBitWidth) == 0 &&
|
|
"Unexpected constant extension");
|
|
|
|
// Ensure every vector element can be represented by the src bitwidth.
|
|
APInt TruncBits = APInt::getZero(NumElts * SrcEltBitWidth);
|
|
for (unsigned I = 0; I != NumElts; ++I) {
|
|
APInt Elt = Bits->extractBits(DstEltBitWidth, I * DstEltBitWidth);
|
|
if ((IsSExt && Elt.getSignificantBits() > SrcEltBitWidth) ||
|
|
(!IsSExt && Elt.getActiveBits() > SrcEltBitWidth))
|
|
return nullptr;
|
|
TruncBits.insertBits(Elt.trunc(SrcEltBitWidth), I * SrcEltBitWidth);
|
|
}
|
|
|
|
Type *Ty = C->getType();
|
|
return rebuildConstant(Ty->getContext(), Ty->getScalarType(), TruncBits,
|
|
SrcEltBitWidth);
|
|
}
|
|
|
|
return nullptr;
|
|
}
|
|
static Constant *rebuildSExtCst(const Constant *C, unsigned NumBits,
|
|
unsigned NumElts, unsigned SrcEltBitWidth) {
|
|
return rebuildExtCst(C, true, NumBits, NumElts, SrcEltBitWidth);
|
|
}
|
|
static Constant *rebuildZExtCst(const Constant *C, unsigned NumBits,
|
|
unsigned NumElts, unsigned SrcEltBitWidth) {
|
|
return rebuildExtCst(C, false, NumBits, NumElts, SrcEltBitWidth);
|
|
}
|
|
|
|
bool X86FixupVectorConstantsPass::processInstruction(MachineFunction &MF,
|
|
MachineBasicBlock &MBB,
|
|
MachineInstr &MI) {
|
|
unsigned Opc = MI.getOpcode();
|
|
MachineConstantPool *CP = MI.getParent()->getParent()->getConstantPool();
|
|
bool HasSSE2 = ST->hasSSE2();
|
|
bool HasSSE41 = ST->hasSSE41();
|
|
bool HasAVX2 = ST->hasAVX2();
|
|
bool HasDQI = ST->hasDQI();
|
|
bool HasBWI = ST->hasBWI();
|
|
bool HasVLX = ST->hasVLX();
|
|
bool MultiDomain = ST->hasAVX512() || ST->hasNoDomainDelayMov();
|
|
bool OptSize = MF.getFunction().hasOptSize();
|
|
|
|
struct FixupEntry {
|
|
int Op;
|
|
int NumCstElts;
|
|
int MemBitWidth;
|
|
std::function<Constant *(const Constant *, unsigned, unsigned, unsigned)>
|
|
RebuildConstant;
|
|
};
|
|
|
|
auto NewOpcPreferable = [&](const FixupEntry &Fixup,
|
|
unsigned RegBitWidth) -> bool {
|
|
if (SM->hasInstrSchedModel()) {
|
|
unsigned NewOpc = Fixup.Op;
|
|
auto *OldDesc = SM->getSchedClassDesc(TII->get(Opc).getSchedClass());
|
|
auto *NewDesc = SM->getSchedClassDesc(TII->get(NewOpc).getSchedClass());
|
|
unsigned BitsSaved = RegBitWidth - (Fixup.NumCstElts * Fixup.MemBitWidth);
|
|
|
|
// Compare tput/lat - avoid any regressions, but allow extra cycle of
|
|
// latency in exchange for each 128-bit (or less) constant pool reduction
|
|
// (this is a very simple cost:benefit estimate - there will probably be
|
|
// better ways to calculate this).
|
|
double OldTput = MCSchedModel::getReciprocalThroughput(*ST, *OldDesc);
|
|
double NewTput = MCSchedModel::getReciprocalThroughput(*ST, *NewDesc);
|
|
if (OldTput != NewTput)
|
|
return NewTput < OldTput;
|
|
|
|
int LatTol = (BitsSaved + 127) / 128;
|
|
int OldLat = MCSchedModel::computeInstrLatency(*ST, *OldDesc);
|
|
int NewLat = MCSchedModel::computeInstrLatency(*ST, *NewDesc);
|
|
if (OldLat != NewLat)
|
|
return NewLat < (OldLat + LatTol);
|
|
}
|
|
|
|
// We either were unable to get tput/lat or all values were equal.
|
|
// Prefer the new opcode for reduced constant pool size.
|
|
return true;
|
|
};
|
|
|
|
auto FixupConstant = [&](ArrayRef<FixupEntry> Fixups, unsigned RegBitWidth,
|
|
unsigned OperandNo) {
|
|
#ifdef EXPENSIVE_CHECKS
|
|
assert(llvm::is_sorted(Fixups,
|
|
[](const FixupEntry &A, const FixupEntry &B) {
|
|
return (A.NumCstElts * A.MemBitWidth) <
|
|
(B.NumCstElts * B.MemBitWidth);
|
|
}) &&
|
|
"Constant fixup table not sorted in ascending constant size");
|
|
#endif
|
|
assert(MI.getNumOperands() >= (OperandNo + X86::AddrNumOperands) &&
|
|
"Unexpected number of operands!");
|
|
if (auto *C = X86::getConstantFromPool(MI, OperandNo)) {
|
|
unsigned CstBitWidth = C->getType()->getPrimitiveSizeInBits();
|
|
RegBitWidth = RegBitWidth ? RegBitWidth : CstBitWidth;
|
|
for (const FixupEntry &Fixup : Fixups) {
|
|
// Always uses the smallest possible constant load with opt/minsize,
|
|
// otherwise use the smallest instruction that doesn't affect
|
|
// performance.
|
|
// TODO: If constant has been hoisted from loop, use smallest constant.
|
|
if (Fixup.Op && (OptSize || NewOpcPreferable(Fixup, RegBitWidth))) {
|
|
// Construct a suitable constant and adjust the MI to use the new
|
|
// constant pool entry.
|
|
if (Constant *NewCst = Fixup.RebuildConstant(
|
|
C, RegBitWidth, Fixup.NumCstElts, Fixup.MemBitWidth)) {
|
|
unsigned NewCPI =
|
|
CP->getConstantPoolIndex(NewCst, Align(Fixup.MemBitWidth / 8));
|
|
MI.setDesc(TII->get(Fixup.Op));
|
|
MI.getOperand(OperandNo + X86::AddrDisp).setIndex(NewCPI);
|
|
return true;
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
|
|
// Attempt to detect a suitable vzload/broadcast/vextload from increasing
|
|
// constant bitwidths. Prefer vzload/broadcast/vextload for same bitwidth:
|
|
// - vzload shouldn't ever need a shuffle port to zero the upper elements and
|
|
// the fp/int domain versions are equally available so we don't introduce a
|
|
// domain crossing penalty.
|
|
// - broadcast sometimes need a shuffle port (especially for 8/16-bit
|
|
// variants), AVX1 only has fp domain broadcasts but AVX2+ have good fp/int
|
|
// domain equivalents.
|
|
// - vextload always needs a shuffle port and is only ever int domain.
|
|
switch (Opc) {
|
|
/* FP Loads */
|
|
case X86::MOVAPDrm:
|
|
case X86::MOVAPSrm:
|
|
case X86::MOVUPDrm:
|
|
case X86::MOVUPSrm: {
|
|
// TODO: SSE3 MOVDDUP Handling
|
|
FixupEntry Fixups[] = {
|
|
{X86::MOVSSrm, 1, 32, rebuildZeroUpperCst},
|
|
{HasSSE2 ? X86::MOVSDrm : 0, 1, 64, rebuildZeroUpperCst}};
|
|
return FixupConstant(Fixups, 128, 1);
|
|
}
|
|
case X86::VMOVAPDrm:
|
|
case X86::VMOVAPSrm:
|
|
case X86::VMOVUPDrm:
|
|
case X86::VMOVUPSrm: {
|
|
FixupEntry Fixups[] = {
|
|
{MultiDomain ? X86::VPMOVSXBQrm : 0, 2, 8, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXBQrm : 0, 2, 8, rebuildZExtCst},
|
|
{X86::VMOVSSrm, 1, 32, rebuildZeroUpperCst},
|
|
{X86::VBROADCASTSSrm, 1, 32, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXBDrm : 0, 4, 8, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXBDrm : 0, 4, 8, rebuildZExtCst},
|
|
{MultiDomain ? X86::VPMOVSXWQrm : 0, 2, 16, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXWQrm : 0, 2, 16, rebuildZExtCst},
|
|
{X86::VMOVSDrm, 1, 64, rebuildZeroUpperCst},
|
|
{X86::VMOVDDUPrm, 1, 64, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXWDrm : 0, 4, 16, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXWDrm : 0, 4, 16, rebuildZExtCst},
|
|
{MultiDomain ? X86::VPMOVSXDQrm : 0, 2, 32, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXDQrm : 0, 2, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 128, 1);
|
|
}
|
|
case X86::VMOVAPDYrm:
|
|
case X86::VMOVAPSYrm:
|
|
case X86::VMOVUPDYrm:
|
|
case X86::VMOVUPSYrm: {
|
|
FixupEntry Fixups[] = {
|
|
{X86::VBROADCASTSSYrm, 1, 32, rebuildSplatCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVSXBQYrm : 0, 4, 8, rebuildSExtCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVZXBQYrm : 0, 4, 8, rebuildZExtCst},
|
|
{X86::VBROADCASTSDYrm, 1, 64, rebuildSplatCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVSXBDYrm : 0, 8, 8, rebuildSExtCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVZXBDYrm : 0, 8, 8, rebuildZExtCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVSXWQYrm : 0, 4, 16, rebuildSExtCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVZXWQYrm : 0, 4, 16, rebuildZExtCst},
|
|
{X86::VBROADCASTF128rm, 1, 128, rebuildSplatCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVSXWDYrm : 0, 8, 16, rebuildSExtCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVZXWDYrm : 0, 8, 16, rebuildZExtCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVSXDQYrm : 0, 4, 32, rebuildSExtCst},
|
|
{HasAVX2 && MultiDomain ? X86::VPMOVZXDQYrm : 0, 4, 32,
|
|
rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 256, 1);
|
|
}
|
|
case X86::VMOVAPDZ128rm:
|
|
case X86::VMOVAPSZ128rm:
|
|
case X86::VMOVUPDZ128rm:
|
|
case X86::VMOVUPSZ128rm: {
|
|
FixupEntry Fixups[] = {
|
|
{MultiDomain ? X86::VPMOVSXBQZ128rm : 0, 2, 8, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXBQZ128rm : 0, 2, 8, rebuildZExtCst},
|
|
{X86::VMOVSSZrm, 1, 32, rebuildZeroUpperCst},
|
|
{X86::VBROADCASTSSZ128rm, 1, 32, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXBDZ128rm : 0, 4, 8, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXBDZ128rm : 0, 4, 8, rebuildZExtCst},
|
|
{MultiDomain ? X86::VPMOVSXWQZ128rm : 0, 2, 16, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXWQZ128rm : 0, 2, 16, rebuildZExtCst},
|
|
{X86::VMOVSDZrm, 1, 64, rebuildZeroUpperCst},
|
|
{X86::VMOVDDUPZ128rm, 1, 64, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXWDZ128rm : 0, 4, 16, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXWDZ128rm : 0, 4, 16, rebuildZExtCst},
|
|
{MultiDomain ? X86::VPMOVSXDQZ128rm : 0, 2, 32, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXDQZ128rm : 0, 2, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 128, 1);
|
|
}
|
|
case X86::VMOVAPDZ256rm:
|
|
case X86::VMOVAPSZ256rm:
|
|
case X86::VMOVUPDZ256rm:
|
|
case X86::VMOVUPSZ256rm: {
|
|
FixupEntry Fixups[] = {
|
|
{X86::VBROADCASTSSZ256rm, 1, 32, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXBQZ256rm : 0, 4, 8, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXBQZ256rm : 0, 4, 8, rebuildZExtCst},
|
|
{X86::VBROADCASTSDZ256rm, 1, 64, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXBDZ256rm : 0, 8, 8, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXBDZ256rm : 0, 8, 8, rebuildZExtCst},
|
|
{MultiDomain ? X86::VPMOVSXWQZ256rm : 0, 4, 16, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXWQZ256rm : 0, 4, 16, rebuildZExtCst},
|
|
{X86::VBROADCASTF32X4Z256rm, 1, 128, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXWDZ256rm : 0, 8, 16, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXWDZ256rm : 0, 8, 16, rebuildZExtCst},
|
|
{MultiDomain ? X86::VPMOVSXDQZ256rm : 0, 4, 32, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXDQZ256rm : 0, 4, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 256, 1);
|
|
}
|
|
case X86::VMOVAPDZrm:
|
|
case X86::VMOVAPSZrm:
|
|
case X86::VMOVUPDZrm:
|
|
case X86::VMOVUPSZrm: {
|
|
FixupEntry Fixups[] = {
|
|
{X86::VBROADCASTSSZrm, 1, 32, rebuildSplatCst},
|
|
{X86::VBROADCASTSDZrm, 1, 64, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXBQZrm : 0, 8, 8, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXBQZrm : 0, 8, 8, rebuildZExtCst},
|
|
{X86::VBROADCASTF32X4Zrm, 1, 128, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXBDZrm : 0, 16, 8, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXBDZrm : 0, 16, 8, rebuildZExtCst},
|
|
{MultiDomain ? X86::VPMOVSXWQZrm : 0, 8, 16, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXWQZrm : 0, 8, 16, rebuildZExtCst},
|
|
{X86::VBROADCASTF64X4Zrm, 1, 256, rebuildSplatCst},
|
|
{MultiDomain ? X86::VPMOVSXWDZrm : 0, 16, 16, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXWDZrm : 0, 16, 16, rebuildZExtCst},
|
|
{MultiDomain ? X86::VPMOVSXDQZrm : 0, 8, 32, rebuildSExtCst},
|
|
{MultiDomain ? X86::VPMOVZXDQZrm : 0, 8, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 512, 1);
|
|
}
|
|
/* Integer Loads */
|
|
case X86::MOVDQArm:
|
|
case X86::MOVDQUrm: {
|
|
FixupEntry Fixups[] = {
|
|
{HasSSE41 ? X86::PMOVSXBQrm : 0, 2, 8, rebuildSExtCst},
|
|
{HasSSE41 ? X86::PMOVZXBQrm : 0, 2, 8, rebuildZExtCst},
|
|
{X86::MOVDI2PDIrm, 1, 32, rebuildZeroUpperCst},
|
|
{HasSSE41 ? X86::PMOVSXBDrm : 0, 4, 8, rebuildSExtCst},
|
|
{HasSSE41 ? X86::PMOVZXBDrm : 0, 4, 8, rebuildZExtCst},
|
|
{HasSSE41 ? X86::PMOVSXWQrm : 0, 2, 16, rebuildSExtCst},
|
|
{HasSSE41 ? X86::PMOVZXWQrm : 0, 2, 16, rebuildZExtCst},
|
|
{X86::MOVQI2PQIrm, 1, 64, rebuildZeroUpperCst},
|
|
{HasSSE41 ? X86::PMOVSXBWrm : 0, 8, 8, rebuildSExtCst},
|
|
{HasSSE41 ? X86::PMOVZXBWrm : 0, 8, 8, rebuildZExtCst},
|
|
{HasSSE41 ? X86::PMOVSXWDrm : 0, 4, 16, rebuildSExtCst},
|
|
{HasSSE41 ? X86::PMOVZXWDrm : 0, 4, 16, rebuildZExtCst},
|
|
{HasSSE41 ? X86::PMOVSXDQrm : 0, 2, 32, rebuildSExtCst},
|
|
{HasSSE41 ? X86::PMOVZXDQrm : 0, 2, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 128, 1);
|
|
}
|
|
case X86::VMOVDQArm:
|
|
case X86::VMOVDQUrm: {
|
|
FixupEntry Fixups[] = {
|
|
{HasAVX2 ? X86::VPBROADCASTBrm : 0, 1, 8, rebuildSplatCst},
|
|
{HasAVX2 ? X86::VPBROADCASTWrm : 0, 1, 16, rebuildSplatCst},
|
|
{X86::VPMOVSXBQrm, 2, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBQrm, 2, 8, rebuildZExtCst},
|
|
{X86::VMOVDI2PDIrm, 1, 32, rebuildZeroUpperCst},
|
|
{HasAVX2 ? X86::VPBROADCASTDrm : X86::VBROADCASTSSrm, 1, 32,
|
|
rebuildSplatCst},
|
|
{X86::VPMOVSXBDrm, 4, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBDrm, 4, 8, rebuildZExtCst},
|
|
{X86::VPMOVSXWQrm, 2, 16, rebuildSExtCst},
|
|
{X86::VPMOVZXWQrm, 2, 16, rebuildZExtCst},
|
|
{X86::VMOVQI2PQIrm, 1, 64, rebuildZeroUpperCst},
|
|
{HasAVX2 ? X86::VPBROADCASTQrm : X86::VMOVDDUPrm, 1, 64,
|
|
rebuildSplatCst},
|
|
{X86::VPMOVSXBWrm, 8, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBWrm, 8, 8, rebuildZExtCst},
|
|
{X86::VPMOVSXWDrm, 4, 16, rebuildSExtCst},
|
|
{X86::VPMOVZXWDrm, 4, 16, rebuildZExtCst},
|
|
{X86::VPMOVSXDQrm, 2, 32, rebuildSExtCst},
|
|
{X86::VPMOVZXDQrm, 2, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 128, 1);
|
|
}
|
|
case X86::VMOVDQAYrm:
|
|
case X86::VMOVDQUYrm: {
|
|
FixupEntry Fixups[] = {
|
|
{HasAVX2 ? X86::VPBROADCASTBYrm : 0, 1, 8, rebuildSplatCst},
|
|
{HasAVX2 ? X86::VPBROADCASTWYrm : 0, 1, 16, rebuildSplatCst},
|
|
{HasAVX2 ? X86::VPBROADCASTDYrm : X86::VBROADCASTSSYrm, 1, 32,
|
|
rebuildSplatCst},
|
|
{HasAVX2 ? X86::VPMOVSXBQYrm : 0, 4, 8, rebuildSExtCst},
|
|
{HasAVX2 ? X86::VPMOVZXBQYrm : 0, 4, 8, rebuildZExtCst},
|
|
{HasAVX2 ? X86::VPBROADCASTQYrm : X86::VBROADCASTSDYrm, 1, 64,
|
|
rebuildSplatCst},
|
|
{HasAVX2 ? X86::VPMOVSXBDYrm : 0, 8, 8, rebuildSExtCst},
|
|
{HasAVX2 ? X86::VPMOVZXBDYrm : 0, 8, 8, rebuildZExtCst},
|
|
{HasAVX2 ? X86::VPMOVSXWQYrm : 0, 4, 16, rebuildSExtCst},
|
|
{HasAVX2 ? X86::VPMOVZXWQYrm : 0, 4, 16, rebuildZExtCst},
|
|
{HasAVX2 ? X86::VBROADCASTI128rm : X86::VBROADCASTF128rm, 1, 128,
|
|
rebuildSplatCst},
|
|
{HasAVX2 ? X86::VPMOVSXBWYrm : 0, 16, 8, rebuildSExtCst},
|
|
{HasAVX2 ? X86::VPMOVZXBWYrm : 0, 16, 8, rebuildZExtCst},
|
|
{HasAVX2 ? X86::VPMOVSXWDYrm : 0, 8, 16, rebuildSExtCst},
|
|
{HasAVX2 ? X86::VPMOVZXWDYrm : 0, 8, 16, rebuildZExtCst},
|
|
{HasAVX2 ? X86::VPMOVSXDQYrm : 0, 4, 32, rebuildSExtCst},
|
|
{HasAVX2 ? X86::VPMOVZXDQYrm : 0, 4, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 256, 1);
|
|
}
|
|
case X86::VMOVDQA32Z128rm:
|
|
case X86::VMOVDQA64Z128rm:
|
|
case X86::VMOVDQU32Z128rm:
|
|
case X86::VMOVDQU64Z128rm: {
|
|
FixupEntry Fixups[] = {
|
|
{HasBWI ? X86::VPBROADCASTBZ128rm : 0, 1, 8, rebuildSplatCst},
|
|
{HasBWI ? X86::VPBROADCASTWZ128rm : 0, 1, 16, rebuildSplatCst},
|
|
{X86::VPMOVSXBQZ128rm, 2, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBQZ128rm, 2, 8, rebuildZExtCst},
|
|
{X86::VMOVDI2PDIZrm, 1, 32, rebuildZeroUpperCst},
|
|
{X86::VPBROADCASTDZ128rm, 1, 32, rebuildSplatCst},
|
|
{X86::VPMOVSXBDZ128rm, 4, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBDZ128rm, 4, 8, rebuildZExtCst},
|
|
{X86::VPMOVSXWQZ128rm, 2, 16, rebuildSExtCst},
|
|
{X86::VPMOVZXWQZ128rm, 2, 16, rebuildZExtCst},
|
|
{X86::VMOVQI2PQIZrm, 1, 64, rebuildZeroUpperCst},
|
|
{X86::VPBROADCASTQZ128rm, 1, 64, rebuildSplatCst},
|
|
{HasBWI ? X86::VPMOVSXBWZ128rm : 0, 8, 8, rebuildSExtCst},
|
|
{HasBWI ? X86::VPMOVZXBWZ128rm : 0, 8, 8, rebuildZExtCst},
|
|
{X86::VPMOVSXWDZ128rm, 4, 16, rebuildSExtCst},
|
|
{X86::VPMOVZXWDZ128rm, 4, 16, rebuildZExtCst},
|
|
{X86::VPMOVSXDQZ128rm, 2, 32, rebuildSExtCst},
|
|
{X86::VPMOVZXDQZ128rm, 2, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 128, 1);
|
|
}
|
|
case X86::VMOVDQA32Z256rm:
|
|
case X86::VMOVDQA64Z256rm:
|
|
case X86::VMOVDQU32Z256rm:
|
|
case X86::VMOVDQU64Z256rm: {
|
|
FixupEntry Fixups[] = {
|
|
{HasBWI ? X86::VPBROADCASTBZ256rm : 0, 1, 8, rebuildSplatCst},
|
|
{HasBWI ? X86::VPBROADCASTWZ256rm : 0, 1, 16, rebuildSplatCst},
|
|
{X86::VPBROADCASTDZ256rm, 1, 32, rebuildSplatCst},
|
|
{X86::VPMOVSXBQZ256rm, 4, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBQZ256rm, 4, 8, rebuildZExtCst},
|
|
{X86::VPBROADCASTQZ256rm, 1, 64, rebuildSplatCst},
|
|
{X86::VPMOVSXBDZ256rm, 8, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBDZ256rm, 8, 8, rebuildZExtCst},
|
|
{X86::VPMOVSXWQZ256rm, 4, 16, rebuildSExtCst},
|
|
{X86::VPMOVZXWQZ256rm, 4, 16, rebuildZExtCst},
|
|
{X86::VBROADCASTI32X4Z256rm, 1, 128, rebuildSplatCst},
|
|
{HasBWI ? X86::VPMOVSXBWZ256rm : 0, 16, 8, rebuildSExtCst},
|
|
{HasBWI ? X86::VPMOVZXBWZ256rm : 0, 16, 8, rebuildZExtCst},
|
|
{X86::VPMOVSXWDZ256rm, 8, 16, rebuildSExtCst},
|
|
{X86::VPMOVZXWDZ256rm, 8, 16, rebuildZExtCst},
|
|
{X86::VPMOVSXDQZ256rm, 4, 32, rebuildSExtCst},
|
|
{X86::VPMOVZXDQZ256rm, 4, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 256, 1);
|
|
}
|
|
case X86::VMOVDQA32Zrm:
|
|
case X86::VMOVDQA64Zrm:
|
|
case X86::VMOVDQU32Zrm:
|
|
case X86::VMOVDQU64Zrm: {
|
|
FixupEntry Fixups[] = {
|
|
{HasBWI ? X86::VPBROADCASTBZrm : 0, 1, 8, rebuildSplatCst},
|
|
{HasBWI ? X86::VPBROADCASTWZrm : 0, 1, 16, rebuildSplatCst},
|
|
{X86::VPBROADCASTDZrm, 1, 32, rebuildSplatCst},
|
|
{X86::VPBROADCASTQZrm, 1, 64, rebuildSplatCst},
|
|
{X86::VPMOVSXBQZrm, 8, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBQZrm, 8, 8, rebuildZExtCst},
|
|
{X86::VBROADCASTI32X4Zrm, 1, 128, rebuildSplatCst},
|
|
{X86::VPMOVSXBDZrm, 16, 8, rebuildSExtCst},
|
|
{X86::VPMOVZXBDZrm, 16, 8, rebuildZExtCst},
|
|
{X86::VPMOVSXWQZrm, 8, 16, rebuildSExtCst},
|
|
{X86::VPMOVZXWQZrm, 8, 16, rebuildZExtCst},
|
|
{X86::VBROADCASTI64X4Zrm, 1, 256, rebuildSplatCst},
|
|
{HasBWI ? X86::VPMOVSXBWZrm : 0, 32, 8, rebuildSExtCst},
|
|
{HasBWI ? X86::VPMOVZXBWZrm : 0, 32, 8, rebuildZExtCst},
|
|
{X86::VPMOVSXWDZrm, 16, 16, rebuildSExtCst},
|
|
{X86::VPMOVZXWDZrm, 16, 16, rebuildZExtCst},
|
|
{X86::VPMOVSXDQZrm, 8, 32, rebuildSExtCst},
|
|
{X86::VPMOVZXDQZrm, 8, 32, rebuildZExtCst}};
|
|
return FixupConstant(Fixups, 512, 1);
|
|
}
|
|
}
|
|
|
|
auto ConvertToBroadcast = [&](unsigned OpSrc, int BW) {
|
|
if (OpSrc) {
|
|
if (const X86FoldTableEntry *Mem2Bcst =
|
|
llvm::lookupBroadcastFoldTableBySize(OpSrc, BW)) {
|
|
unsigned OpBcst = Mem2Bcst->DstOp;
|
|
unsigned OpNoBcst = Mem2Bcst->Flags & TB_INDEX_MASK;
|
|
FixupEntry Fixups[] = {{(int)OpBcst, 1, BW, rebuildSplatCst}};
|
|
// TODO: Add support for RegBitWidth, but currently rebuildSplatCst
|
|
// doesn't require it (defaults to Constant::getPrimitiveSizeInBits).
|
|
return FixupConstant(Fixups, 0, OpNoBcst);
|
|
}
|
|
}
|
|
return false;
|
|
};
|
|
|
|
// Attempt to find a AVX512 mapping from a full width memory-fold instruction
|
|
// to a broadcast-fold instruction variant.
|
|
if ((MI.getDesc().TSFlags & X86II::EncodingMask) == X86II::EVEX)
|
|
return ConvertToBroadcast(Opc, 32) || ConvertToBroadcast(Opc, 64);
|
|
|
|
// Reverse the X86InstrInfo::setExecutionDomainCustom EVEX->VEX logic
|
|
// conversion to see if we can convert to a broadcasted (integer) logic op.
|
|
if (HasVLX && !HasDQI) {
|
|
unsigned OpSrc32 = 0, OpSrc64 = 0;
|
|
switch (Opc) {
|
|
case X86::VANDPDrm:
|
|
case X86::VANDPSrm:
|
|
case X86::VPANDrm:
|
|
OpSrc32 = X86 ::VPANDDZ128rm;
|
|
OpSrc64 = X86 ::VPANDQZ128rm;
|
|
break;
|
|
case X86::VANDPDYrm:
|
|
case X86::VANDPSYrm:
|
|
case X86::VPANDYrm:
|
|
OpSrc32 = X86 ::VPANDDZ256rm;
|
|
OpSrc64 = X86 ::VPANDQZ256rm;
|
|
break;
|
|
case X86::VANDNPDrm:
|
|
case X86::VANDNPSrm:
|
|
case X86::VPANDNrm:
|
|
OpSrc32 = X86 ::VPANDNDZ128rm;
|
|
OpSrc64 = X86 ::VPANDNQZ128rm;
|
|
break;
|
|
case X86::VANDNPDYrm:
|
|
case X86::VANDNPSYrm:
|
|
case X86::VPANDNYrm:
|
|
OpSrc32 = X86 ::VPANDNDZ256rm;
|
|
OpSrc64 = X86 ::VPANDNQZ256rm;
|
|
break;
|
|
case X86::VORPDrm:
|
|
case X86::VORPSrm:
|
|
case X86::VPORrm:
|
|
OpSrc32 = X86 ::VPORDZ128rm;
|
|
OpSrc64 = X86 ::VPORQZ128rm;
|
|
break;
|
|
case X86::VORPDYrm:
|
|
case X86::VORPSYrm:
|
|
case X86::VPORYrm:
|
|
OpSrc32 = X86 ::VPORDZ256rm;
|
|
OpSrc64 = X86 ::VPORQZ256rm;
|
|
break;
|
|
case X86::VXORPDrm:
|
|
case X86::VXORPSrm:
|
|
case X86::VPXORrm:
|
|
OpSrc32 = X86 ::VPXORDZ128rm;
|
|
OpSrc64 = X86 ::VPXORQZ128rm;
|
|
break;
|
|
case X86::VXORPDYrm:
|
|
case X86::VXORPSYrm:
|
|
case X86::VPXORYrm:
|
|
OpSrc32 = X86 ::VPXORDZ256rm;
|
|
OpSrc64 = X86 ::VPXORQZ256rm;
|
|
break;
|
|
}
|
|
if (OpSrc32 || OpSrc64)
|
|
return ConvertToBroadcast(OpSrc32, 32) || ConvertToBroadcast(OpSrc64, 64);
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
bool X86FixupVectorConstantsPass::runOnMachineFunction(MachineFunction &MF) {
|
|
LLVM_DEBUG(dbgs() << "Start X86FixupVectorConstants\n";);
|
|
bool Changed = false;
|
|
ST = &MF.getSubtarget<X86Subtarget>();
|
|
TII = ST->getInstrInfo();
|
|
SM = &ST->getSchedModel();
|
|
|
|
for (MachineBasicBlock &MBB : MF) {
|
|
for (MachineInstr &MI : MBB) {
|
|
if (processInstruction(MF, MBB, MI)) {
|
|
++NumInstChanges;
|
|
Changed = true;
|
|
}
|
|
}
|
|
}
|
|
LLVM_DEBUG(dbgs() << "End X86FixupVectorConstants\n";);
|
|
return Changed;
|
|
}
|