diff --git a/llvm/include/llvm/Analysis/AliasAnalysis.h b/llvm/include/llvm/Analysis/AliasAnalysis.h index acc580f92b40..b192a9f5e65e 100644 --- a/llvm/include/llvm/Analysis/AliasAnalysis.h +++ b/llvm/include/llvm/Analysis/AliasAnalysis.h @@ -643,6 +643,9 @@ public: bool pointsToConstantMemory(const MemoryLocation &Loc, bool OrLocal = false) { return isNoModRef(AA.getModRefInfoMask(Loc, AAQI, OrLocal)); } + bool pointsToConstantMemory(const Value *P, bool OrLocal = false) { + return pointsToConstantMemory(MemoryLocation::getBeforeOrAfter(P), OrLocal); + } ModRefInfo getModRefInfoMask(const MemoryLocation &Loc, bool IgnoreLocals = false) { return AA.getModRefInfoMask(Loc, AAQI, IgnoreLocals); @@ -668,6 +671,9 @@ public: MemoryLocation(V2, LocationSize::precise(1))) == AliasResult::MustAlias; } + bool isNoAlias(const MemoryLocation &LocA, const MemoryLocation &LocB) { + return alias(LocA, LocB) == AliasResult::NoAlias; + } ModRefInfo callCapturesBefore(const Instruction *I, const MemoryLocation &MemLoc, DominatorTree *DT) { diff --git a/llvm/include/llvm/CodeGen/MachineInstr.h b/llvm/include/llvm/CodeGen/MachineInstr.h index efac83d9e1c9..109aac44b866 100644 --- a/llvm/include/llvm/CodeGen/MachineInstr.h +++ b/llvm/include/llvm/CodeGen/MachineInstr.h @@ -42,6 +42,7 @@ class DILabel; class Instruction; class MDNode; class AAResults; +class BatchAAResults; template class ArrayRef; class DIExpression; class DILocalVariable; @@ -1753,6 +1754,8 @@ public: /// @param AA Optional alias analysis, used to compare memory operands. /// @param Other MachineInstr to check aliasing against. /// @param UseTBAA Whether to pass TBAA information to alias analysis. + bool mayAlias(BatchAAResults *AA, const MachineInstr &Other, + bool UseTBAA) const; bool mayAlias(AAResults *AA, const MachineInstr &Other, bool UseTBAA) const; /// Return true if this instruction may have an ordered diff --git a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h index 822b06f080fa..aaa10e684687 100644 --- a/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h +++ b/llvm/include/llvm/CodeGen/ScheduleDAGInstrs.h @@ -19,6 +19,7 @@ #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/SparseMultiSet.h" #include "llvm/ADT/identity.h" +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/LiveRegUnits.h" #include "llvm/CodeGen/MachineBasicBlock.h" #include "llvm/CodeGen/ScheduleDAG.h" @@ -169,7 +170,7 @@ namespace llvm { /// Tracks the last instructions in this region using each virtual register. VReg2SUnitOperIdxMultiMap CurrentVRegUses; - AAResults *AAForDep = nullptr; + mutable std::optional AAForDep; /// Remember a generic side-effecting instruction as we proceed. /// No other SU ever gets scheduled around it (except in the special @@ -201,6 +202,13 @@ namespace llvm { /// a means of remembering which SUs depend on which memory locations. class Value2SUsMap; + /// Returns a (possibly null) pointer to the current BatchAAResults. + BatchAAResults *getAAForDep() const { + if (AAForDep.has_value()) + return &AAForDep.value(); + return nullptr; + } + /// Reduces maps in FIFO order, by N SUs. This is better than turning /// every Nth memory SU into BarrierChain in buildSchedGraph(), since /// it avoids unnecessary edges between seen SUs above the new BarrierChain, diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index ba0538f7084e..461c0c1ead16 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -61,7 +61,7 @@ class Type; template struct GraphTraits; template class SmallSetVector; template struct FoldingSetTrait; -class AAResults; +class BatchAAResults; class BlockAddress; class BlockFrequencyInfo; class Constant; @@ -602,7 +602,8 @@ public: /// certain types of nodes together, or eliminating superfluous nodes. The /// Level argument controls whether Combine is allowed to produce nodes and /// types that are illegal on the target. - void Combine(CombineLevel Level, AAResults *AA, CodeGenOptLevel OptLevel); + void Combine(CombineLevel Level, BatchAAResults *BatchAA, + CodeGenOptLevel OptLevel); /// This transforms the SelectionDAG into a SelectionDAG that /// only uses types natively supported by the target. @@ -1202,12 +1203,14 @@ public: /* \p CI if not null is the memset call being lowered. * \p OverrideTailCall is an optional parameter that can be used to override * the tail call optimization decision. */ - SDValue - getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, - SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, - const CallInst *CI, std::optional OverrideTailCall, - MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, - const AAMDNodes &AAInfo = AAMDNodes(), AAResults *AA = nullptr); + SDValue getMemcpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, + SDValue Size, Align Alignment, bool isVol, + bool AlwaysInline, const CallInst *CI, + std::optional OverrideTailCall, + MachinePointerInfo DstPtrInfo, + MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo = AAMDNodes(), + BatchAAResults *BatchAA = nullptr); /* \p CI if not null is the memset call being lowered. * \p OverrideTailCall is an optional parameter that can be used to override @@ -1218,7 +1221,7 @@ public: MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo = AAMDNodes(), - AAResults *AA = nullptr); + BatchAAResults *BatchAA = nullptr); SDValue getMemset(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGISel.h b/llvm/include/llvm/CodeGen/SelectionDAGISel.h index 43ba8f4c44cf..e9452a6dc623 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGISel.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGISel.h @@ -14,6 +14,7 @@ #ifndef LLVM_CODEGEN_SELECTIONDAGISEL_H #define LLVM_CODEGEN_SELECTIONDAGISEL_H +#include "llvm/Analysis/AliasAnalysis.h" #include "llvm/CodeGen/MachineFunctionPass.h" #include "llvm/CodeGen/MachinePassManager.h" #include "llvm/CodeGen/SelectionDAG.h" @@ -52,7 +53,7 @@ public: MachineRegisterInfo *RegInfo; SelectionDAG *CurDAG; std::unique_ptr SDB; - AAResults *AA = nullptr; + mutable std::optional BatchAA; AssumptionCache *AC = nullptr; GCFunctionInfo *GFI = nullptr; SSPLayoutInfo *SP = nullptr; @@ -81,6 +82,13 @@ public: CodeGenOptLevel OL = CodeGenOptLevel::Default); virtual ~SelectionDAGISel(); + /// Returns a (possibly null) pointer to the current BatchAAResults. + BatchAAResults *getBatchAA() const { + if (BatchAA.has_value()) + return &BatchAA.value(); + return nullptr; + } + const TargetLowering *getTargetLowering() const { return TLI; } void initializeAnalysisResults(MachineFunctionAnalysisManager &MFAM); diff --git a/llvm/lib/CodeGen/MachineInstr.cpp b/llvm/lib/CodeGen/MachineInstr.cpp index ef36dfc47219..8c2fab18a24c 100644 --- a/llvm/lib/CodeGen/MachineInstr.cpp +++ b/llvm/lib/CodeGen/MachineInstr.cpp @@ -1350,8 +1350,9 @@ bool MachineInstr::wouldBeTriviallyDead() const { return isPHI() || isSafeToMove(SawStore); } -static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, - bool UseTBAA, const MachineMemOperand *MMOa, +static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, + BatchAAResults *AA, bool UseTBAA, + const MachineMemOperand *MMOa, const MachineMemOperand *MMOb) { // The following interface to AA is fashioned after DAGCombiner::isAlias and // operates with MachineMemOperand offset with some important assumptions: @@ -1434,7 +1435,7 @@ static bool MemOperandsHaveAlias(const MachineFrameInfo &MFI, AAResults *AA, MemoryLocation(ValB, LocB, UseTBAA ? MMOb->getAAInfo() : AAMDNodes())); } -bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other, +bool MachineInstr::mayAlias(BatchAAResults *AA, const MachineInstr &Other, bool UseTBAA) const { const MachineFunction *MF = getMF(); const TargetInstrInfo *TII = MF->getSubtarget().getInstrInfo(); @@ -1478,6 +1479,15 @@ bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other, return false; } +bool MachineInstr::mayAlias(AAResults *AA, const MachineInstr &Other, + bool UseTBAA) const { + if (AA) { + BatchAAResults BAA(*AA); + return mayAlias(&BAA, Other, UseTBAA); + } + return mayAlias(static_cast(nullptr), Other, UseTBAA); +} + /// hasOrderedMemoryRef - Return true if this instruction may have an ordered /// or volatile memory reference, or if the information describing the memory /// reference is not available. Return false if it is known to have no ordered diff --git a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp index 8e3e06bf5715..cc98c52e90ea 100644 --- a/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp +++ b/llvm/lib/CodeGen/ScheduleDAGInstrs.cpp @@ -551,7 +551,7 @@ void ScheduleDAGInstrs::addVRegUseDeps(SUnit *SU, unsigned OperIdx) { void ScheduleDAGInstrs::addChainDependency (SUnit *SUa, SUnit *SUb, unsigned Latency) { - if (SUa->getInstr()->mayAlias(AAForDep, *SUb->getInstr(), UseTBAA)) { + if (SUa->getInstr()->mayAlias(getAAForDep(), *SUb->getInstr(), UseTBAA)) { SDep Dep(SUa, SDep::MayAliasMem); Dep.setLatency(Latency); SUb->addPred(Dep); @@ -740,7 +740,8 @@ void ScheduleDAGInstrs::buildSchedGraph(AAResults *AA, const TargetSubtargetInfo &ST = MF.getSubtarget(); bool UseAA = EnableAASchedMI.getNumOccurrences() > 0 ? EnableAASchedMI : ST.useAA(); - AAForDep = UseAA ? AA : nullptr; + if (UseAA && AA) + AAForDep.emplace(*AA); BarrierChain = nullptr; diff --git a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp index 21d5e0a1b295..a0c703d2df8a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/DAGCombiner.cpp @@ -191,8 +191,8 @@ namespace { /// candidate again. DenseMap> StoreRootCountMap; - // AA - Used for DAG load/store alias analysis. - AliasAnalysis *AA; + // BatchAA - Used for DAG load/store alias analysis. + BatchAAResults *BatchAA; /// This caches all chains that have already been processed in /// DAGCombiner::getStoreMergeCandidates() and found to have no mergeable @@ -247,9 +247,10 @@ namespace { SDValue visit(SDNode *N); public: - DAGCombiner(SelectionDAG &D, AliasAnalysis *AA, CodeGenOptLevel OL) + DAGCombiner(SelectionDAG &D, BatchAAResults *BatchAA, CodeGenOptLevel OL) : DAG(D), TLI(D.getTargetLoweringInfo()), - STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), AA(AA) { + STI(D.getSubtarget().getSelectionDAGInfo()), OptLevel(OL), + BatchAA(BatchAA) { ForCodeSize = DAG.shouldOptForSize(); DisableGenericCombines = STI && STI->disableGenericCombines(OptLevel); @@ -28918,7 +28919,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { UseAA = false; #endif - if (UseAA && AA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && + if (UseAA && BatchAA && MUC0.MMO->getValue() && MUC1.MMO->getValue() && Size0.hasValue() && Size1.hasValue() && // Can't represent a scalable size + fixed offset in LocationSize (!Size0.isScalable() || SrcValOffset0 == 0) && @@ -28933,7 +28934,7 @@ bool DAGCombiner::mayAlias(SDNode *Op0, SDNode *Op1) const { Size0.isScalable() ? Size0 : LocationSize::precise(Overlap0); LocationSize Loc1 = Size1.isScalable() ? Size1 : LocationSize::precise(Overlap1); - if (AA->isNoAlias( + if (BatchAA->isNoAlias( MemoryLocation(MUC0.MMO->getValue(), Loc0, UseTBAA ? MUC0.MMO->getAAInfo() : AAMDNodes()), MemoryLocation(MUC1.MMO->getValue(), Loc1, @@ -29239,8 +29240,8 @@ bool DAGCombiner::findBetterNeighborChains(StoreSDNode *St) { } /// This is the entry point for the file. -void SelectionDAG::Combine(CombineLevel Level, AliasAnalysis *AA, +void SelectionDAG::Combine(CombineLevel Level, BatchAAResults *BatchAA, CodeGenOptLevel OptLevel) { /// This is the main entry point to this class. - DAGCombiner(*this, AA, OptLevel).Run(Level); + DAGCombiner(*this, BatchAA, OptLevel).Run(Level); } diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp index 26eba4b257fb..fd4641ec6f12 100644 --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGFast.cpp @@ -118,7 +118,7 @@ void ScheduleDAGFast::Schedule() { LiveRegCycles.resize(TRI->getNumRegs(), 0); // Build the scheduling graph. - BuildSchedGraph(nullptr); + BuildSchedGraph(); LLVM_DEBUG(dump()); diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp index 51ee3cc681f0..436c42f7e18f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGRRList.cpp @@ -370,7 +370,7 @@ void ScheduleDAGRRList::Schedule() { assert(Interferences.empty() && LRegsMap.empty() && "stale Interferences"); // Build the scheduling graph. - BuildSchedGraph(nullptr); + BuildSchedGraph(); LLVM_DEBUG(dump()); Topo.MarkDirty(); diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp index ac6c44ec6354..d04bd6e98097 100644 --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.cpp @@ -536,7 +536,7 @@ void ScheduleDAGSDNodes::AddSchedEdges() { /// are input. This SUnit graph is similar to the SelectionDAG, but /// excludes nodes that aren't interesting to scheduling, and represents /// glued together nodes with a single SUnit. -void ScheduleDAGSDNodes::BuildSchedGraph(AAResults *AA) { +void ScheduleDAGSDNodes::BuildSchedGraph() { // Cluster certain nodes which should be scheduled together. ClusterNodes(); // Populate the SUnits array. diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h index b7d25c6ccc9b..ff5615b7658f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGSDNodes.h @@ -94,7 +94,7 @@ class InstrItineraryData; /// are input. This SUnit graph is similar to the SelectionDAG, but /// excludes nodes that aren't interesting to scheduling, and represents /// flagged together nodes with a single SUnit. - void BuildSchedGraph(AAResults *AA); + void BuildSchedGraph(); /// InitNumRegDefsLeft - Determine the # of regs defined by this node. /// diff --git a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp index ae42a870ea2f..def0f9589f3f 100644 --- a/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/ScheduleDAGVLIW.cpp @@ -59,14 +59,10 @@ private: /// HazardRec - The hazard recognizer to use. ScheduleHazardRecognizer *HazardRec; - /// AA - AAResults for making memory reference queries. - AAResults *AA; - public: - ScheduleDAGVLIW(MachineFunction &mf, AAResults *aa, - SchedulingPriorityQueue *availqueue) - : ScheduleDAGSDNodes(mf), AvailableQueue(availqueue), AA(aa) { - const TargetSubtargetInfo &STI = mf.getSubtarget(); + ScheduleDAGVLIW(MachineFunction &MF, SchedulingPriorityQueue *AvailableQueue) + : ScheduleDAGSDNodes(MF), AvailableQueue(AvailableQueue) { + const TargetSubtargetInfo &STI = MF.getSubtarget(); HazardRec = STI.getInstrInfo()->CreateTargetHazardRecognizer(&STI, this); } @@ -91,7 +87,7 @@ void ScheduleDAGVLIW::Schedule() { << " '" << BB->getName() << "' **********\n"); // Build the scheduling graph. - BuildSchedGraph(AA); + BuildSchedGraph(); AvailableQueue->initNodes(SUnits); @@ -267,5 +263,5 @@ void ScheduleDAGVLIW::listScheduleTopDown() { /// createVLIWDAGScheduler - This creates a top-down list scheduler. ScheduleDAGSDNodes *llvm::createVLIWDAGScheduler(SelectionDAGISel *IS, CodeGenOptLevel) { - return new ScheduleDAGVLIW(*IS->MF, IS->AA, new ResourcePriorityQueue(IS)); + return new ScheduleDAGVLIW(*IS->MF, new ResourcePriorityQueue(IS)); } diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index 743ae4895a1b..0f9790a10a13 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -8126,13 +8126,11 @@ static void chainLoadsAndStoresForMemcpy(SelectionDAG &DAG, const SDLoc &dl, } } -static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, - SDValue Chain, SDValue Dst, SDValue Src, - uint64_t Size, Align Alignment, - bool isVol, bool AlwaysInline, - MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo, - const AAMDNodes &AAInfo, AAResults *AA) { +static SDValue getMemcpyLoadsAndStores( + SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, + uint64_t Size, Align Alignment, bool isVol, bool AlwaysInline, + MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, + const AAMDNodes &AAInfo, BatchAAResults *BatchAA) { // Turn a memcpy of undef to nop. // FIXME: We need to honor volatile even is Src is undef. if (Src.isUndef()) @@ -8198,8 +8196,8 @@ static SDValue getMemcpyLoadsAndStores(SelectionDAG &DAG, const SDLoc &dl, const Value *SrcVal = dyn_cast_if_present(SrcPtrInfo.V); bool isConstant = - AA && SrcVal && - AA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo)); + BatchAA && SrcVal && + BatchAA->pointsToConstantMemory(MemoryLocation(SrcVal, Size, AAInfo)); MachineMemOperand::Flags MMOFlags = isVol ? MachineMemOperand::MOVolatile : MachineMemOperand::MONone; @@ -8584,7 +8582,8 @@ SDValue SelectionDAG::getMemcpy( SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, SDValue Size, Align Alignment, bool isVol, bool AlwaysInline, const CallInst *CI, std::optional OverrideTailCall, MachinePointerInfo DstPtrInfo, - MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, AAResults *AA) { + MachinePointerInfo SrcPtrInfo, const AAMDNodes &AAInfo, + BatchAAResults *BatchAA) { // Check to see if we should lower the memcpy to loads and stores first. // For cases within the target-specified limits, this is the best choice. ConstantSDNode *ConstantSize = dyn_cast(Size); @@ -8595,7 +8594,7 @@ SDValue SelectionDAG::getMemcpy( SDValue Result = getMemcpyLoadsAndStores( *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, AA); + isVol, false, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA); if (Result.getNode()) return Result; } @@ -8616,7 +8615,7 @@ SDValue SelectionDAG::getMemcpy( assert(ConstantSize && "AlwaysInline requires a constant size!"); return getMemcpyLoadsAndStores( *this, dl, Chain, Dst, Src, ConstantSize->getZExtValue(), Alignment, - isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, AA); + isVol, true, DstPtrInfo, SrcPtrInfo, AAInfo, BatchAA); } checkAddrSpaceIsValidForLibcall(TLI, DstPtrInfo.getAddrSpace()); @@ -8711,7 +8710,8 @@ SDValue SelectionDAG::getMemmove(SDValue Chain, const SDLoc &dl, SDValue Dst, std::optional OverrideTailCall, MachinePointerInfo DstPtrInfo, MachinePointerInfo SrcPtrInfo, - const AAMDNodes &AAInfo, AAResults *AA) { + const AAMDNodes &AAInfo, + BatchAAResults *BatchAA) { // Check to see if we should lower the memmove to loads and stores first. // For cases within the target-specified limits, this is the best choice. ConstantSDNode *ConstantSize = dyn_cast(Size); diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index 9f1aadcb279a..700eb26ca02a 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -1082,10 +1082,10 @@ RegsForValue::getRegsAndSizes() const { return OutVec; } -void SelectionDAGBuilder::init(GCFunctionInfo *gfi, AliasAnalysis *aa, +void SelectionDAGBuilder::init(GCFunctionInfo *gfi, BatchAAResults *aa, AssumptionCache *ac, const TargetLibraryInfo *li) { - AA = aa; + BatchAA = aa; AC = ac; GFI = gfi; LibInfo = li; @@ -4585,8 +4585,8 @@ void SelectionDAGBuilder::visitLoad(const LoadInst &I) { Root = getRoot(); else if (NumValues > MaxParallelChains) Root = getMemoryRoot(); - else if (AA && - AA->pointsToConstantMemory(MemoryLocation( + else if (BatchAA && + BatchAA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), AAInfo))) { @@ -4688,8 +4688,8 @@ void SelectionDAGBuilder::visitLoadFromSwiftError(const LoadInst &I) { const Value *SV = I.getOperand(0); Type *Ty = I.getType(); assert( - (!AA || - !AA->pointsToConstantMemory(MemoryLocation( + (!BatchAA || + !BatchAA->pointsToConstantMemory(MemoryLocation( SV, LocationSize::precise(DAG.getDataLayout().getTypeStoreSize(Ty)), I.getAAMetadata()))) && "load_from_swift_error should not be constant memory"); @@ -4998,7 +4998,7 @@ void SelectionDAGBuilder::visitMaskedLoad(const CallInst &I, bool IsExpanding) { // Do not serialize masked loads of constant memory with anything. MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); - bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); + bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML); SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); @@ -6534,7 +6534,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, /* AlwaysInline */ false, &I, std::nullopt, MachinePointerInfo(I.getArgOperand(0)), MachinePointerInfo(I.getArgOperand(1)), - I.getAAMetadata(), AA); + I.getAAMetadata(), BatchAA); updateDAGForMaybeTailCall(MC); return; } @@ -6555,7 +6555,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, /* AlwaysInline */ true, &I, std::nullopt, MachinePointerInfo(I.getArgOperand(0)), MachinePointerInfo(I.getArgOperand(1)), - I.getAAMetadata(), AA); + I.getAAMetadata(), BatchAA); updateDAGForMaybeTailCall(MC); return; } @@ -6608,7 +6608,7 @@ void SelectionDAGBuilder::visitIntrinsicCall(const CallInst &I, /* OverrideTailCall */ std::nullopt, MachinePointerInfo(I.getArgOperand(0)), MachinePointerInfo(I.getArgOperand(1)), - I.getAAMetadata(), AA); + I.getAAMetadata(), BatchAA); updateDAGForMaybeTailCall(MM); return; } @@ -8435,7 +8435,7 @@ void SelectionDAGBuilder::visitVPLoad( if (!Alignment) Alignment = DAG.getEVTAlign(VT); MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); - bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); + bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML); SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( MachinePointerInfo(PtrOperand), MachineMemOperand::MOLoad, @@ -8564,7 +8564,7 @@ void SelectionDAGBuilder::visitVPStridedLoad( AAMDNodes AAInfo = VPIntrin.getAAMetadata(); const MDNode *Ranges = getRangeMetadata(VPIntrin); MemoryLocation ML = MemoryLocation::getAfter(PtrOperand, AAInfo); - bool AddToChain = !AA || !AA->pointsToConstantMemory(ML); + bool AddToChain = !BatchAA || !BatchAA->pointsToConstantMemory(ML); SDValue InChain = AddToChain ? DAG.getRoot() : DAG.getEntryNode(); unsigned AS = PtrOperand->getType()->getPointerAddressSpace(); MachineMemOperand *MMO = DAG.getMachineFunction().getMachineMemOperand( @@ -9021,7 +9021,7 @@ static SDValue getMemCmpLoad(const Value *PtrVal, MVT LoadVT, bool ConstantMemory = false; // Do not serialize (non-volatile) loads of constant memory with anything. - if (Builder.AA && Builder.AA->pointsToConstantMemory(PtrVal)) { + if (Builder.BatchAA && Builder.BatchAA->pointsToConstantMemory(PtrVal)) { Root = Builder.DAG.getEntryNode(); ConstantMemory = true; } else { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index 3a8dc25e9870..ed85deef64fa 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -225,7 +225,7 @@ public: static const unsigned LowestSDNodeOrder = 1; SelectionDAG &DAG; - AAResults *AA = nullptr; + BatchAAResults *BatchAA = nullptr; AssumptionCache *AC = nullptr; const TargetLibraryInfo *LibInfo = nullptr; @@ -280,7 +280,7 @@ public: SL(std::make_unique(this, funcinfo)), FuncInfo(funcinfo), SwiftError(swifterror) {} - void init(GCFunctionInfo *gfi, AAResults *AA, AssumptionCache *AC, + void init(GCFunctionInfo *gfi, BatchAAResults *BatchAA, AssumptionCache *AC, const TargetLibraryInfo *li); /// Clear out the current SelectionDAG and the associated state and prepare diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp index b416e98fe61a..3b1abf7f3d99 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGISel.cpp @@ -502,9 +502,9 @@ void SelectionDAGISel::initializeAnalysisResults( FuncInfo->BPI = nullptr; if (OptLevel != CodeGenOptLevel::None) - AA = &FAM.getResult(Fn); + BatchAA.emplace(FAM.getResult(Fn)); else - AA = nullptr; + BatchAA = std::nullopt; SP = &FAM.getResult(Fn); @@ -560,9 +560,9 @@ void SelectionDAGISel::initializeAnalysisResults(MachineFunctionPass &MFP) { FuncInfo->BPI = nullptr; if (OptLevel != CodeGenOptLevel::None) - AA = &MFP.getAnalysis().getAAResults(); + BatchAA.emplace(MFP.getAnalysis().getAAResults()); else - AA = nullptr; + BatchAA = std::nullopt; SP = &MFP.getAnalysis().getLayoutInfo(); @@ -581,7 +581,7 @@ bool SelectionDAGISel::runOnMachineFunction(MachineFunction &mf) { ISEL_DUMP(dbgs() << "\n\n\n=== " << FuncName << '\n'); - SDB->init(GFI, AA, AC, LibInfo); + SDB->init(GFI, getBatchAA(), AC, LibInfo); MF->setHasInlineAsm(false); @@ -955,7 +955,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { { NamedRegionTimer T("combine1", "DAG Combining 1", GroupName, GroupDescription, TimePassesIsEnabled); - CurDAG->Combine(BeforeLegalizeTypes, AA, OptLevel); + CurDAG->Combine(BeforeLegalizeTypes, getBatchAA(), OptLevel); } ISEL_DUMP(dbgs() << "\nOptimized lowered selection DAG: " @@ -1001,7 +1001,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { { NamedRegionTimer T("combine_lt", "DAG Combining after legalize types", GroupName, GroupDescription, TimePassesIsEnabled); - CurDAG->Combine(AfterLegalizeTypes, AA, OptLevel); + CurDAG->Combine(AfterLegalizeTypes, getBatchAA(), OptLevel); } ISEL_DUMP(dbgs() << "\nOptimized type-legalized selection DAG: " @@ -1055,7 +1055,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { { NamedRegionTimer T("combine_lv", "DAG Combining after legalize vectors", GroupName, GroupDescription, TimePassesIsEnabled); - CurDAG->Combine(AfterLegalizeVectorOps, AA, OptLevel); + CurDAG->Combine(AfterLegalizeVectorOps, getBatchAA(), OptLevel); } ISEL_DUMP(dbgs() << "\nOptimized vector-legalized selection DAG: " @@ -1095,7 +1095,7 @@ void SelectionDAGISel::CodeGenAndEmitDAG() { { NamedRegionTimer T("combine2", "DAG Combining 2", GroupName, GroupDescription, TimePassesIsEnabled); - CurDAG->Combine(AfterLegalizeDAG, AA, OptLevel); + CurDAG->Combine(AfterLegalizeDAG, getBatchAA(), OptLevel); } ISEL_DUMP(dbgs() << "\nOptimized legalized selection DAG: " diff --git a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp index 3d90e3f6f678..caf01ccd1ef7 100644 --- a/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp +++ b/llvm/lib/Target/SystemZ/SystemZISelDAGToDAG.cpp @@ -1498,8 +1498,8 @@ bool SystemZDAGToDAGISel::canUseBlockOperation(StoreSDNode *Store, if (V1 == V2 && End1 == End2) return false; - return AA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()), - MemoryLocation(V2, End2, Store->getAAInfo())); + return BatchAA->isNoAlias(MemoryLocation(V1, End1, Load->getAAInfo()), + MemoryLocation(V2, End2, Store->getAAInfo())); } bool SystemZDAGToDAGISel::storeLoadCanUseMVC(SDNode *N) const {