diff --git a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp index d60c9f65e3ac..583ba283b07e 100644 --- a/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp +++ b/llvm/lib/Transforms/Vectorize/SLPVectorizer.cpp @@ -2108,7 +2108,7 @@ public: } /// Checks if the provided list of reduced values was checked already for /// vectorization. - bool areAnalyzedReductionVals(ArrayRef VL) { + bool areAnalyzedReductionVals(ArrayRef VL) const { return AnalyzedReductionVals.contains(hash_value(VL)); } /// Adds the list of reduced values to list of already checked values for the @@ -3539,6 +3539,24 @@ namespace { enum class LoadsState { Gather, Vectorize, ScatterVectorize }; } // anonymous namespace +static bool arePointersCompatible(Value *Ptr1, Value *Ptr2, + bool CompareOpcodes = true) { + if (getUnderlyingObject(Ptr1) != getUnderlyingObject(Ptr2)) + return false; + auto *GEP1 = dyn_cast(Ptr1); + if (!GEP1) + return false; + auto *GEP2 = dyn_cast(Ptr2); + if (!GEP2) + return false; + return GEP1->getNumOperands() == 2 && GEP2->getNumOperands() == 2 && + ((isConstant(GEP1->getOperand(1)) && + isConstant(GEP2->getOperand(1))) || + !CompareOpcodes || + getSameOpcode({GEP1->getOperand(1), GEP2->getOperand(1)}) + .getOpcode()); +} + /// Checks if the given array of loads can be represented as a vectorized, /// scatter or just simple gather. static LoadsState canVectorizeLoads(ArrayRef VL, const Value *VL0, @@ -3575,17 +3593,7 @@ static LoadsState canVectorizeLoads(ArrayRef VL, const Value *VL0, // Check the order of pointer operands or that all pointers are the same. bool IsSorted = sortPtrAccesses(PointerOps, ScalarTy, DL, SE, Order); if (IsSorted || all_of(PointerOps, [&PointerOps](Value *P) { - if (getUnderlyingObject(P) != getUnderlyingObject(PointerOps.front())) - return false; - auto *GEP = dyn_cast(P); - if (!GEP) - return false; - auto *GEP0 = cast(PointerOps.front()); - return GEP->getNumOperands() == 2 && - ((isConstant(GEP->getOperand(1)) && - isConstant(GEP0->getOperand(1))) || - getSameOpcode({GEP->getOperand(1), GEP0->getOperand(1)}) - .getOpcode()); + return arePointersCompatible(P, PointerOps.front()); })) { if (IsSorted) { Value *Ptr0; @@ -4628,11 +4636,11 @@ static std::pair generateKeySubkey( hash_code SubKey = hash_value(0); // Sort the loads by the distance between the pointers. if (auto *LI = dyn_cast(V)) { - Key = hash_combine(hash_value(Instruction::Load), Key); + Key = hash_combine(LI->getType(), hash_value(Instruction::Load), Key); if (LI->isSimple()) SubKey = hash_value(LoadsSubkeyGenerator(Key, LI)); else - SubKey = hash_value(LI); + Key = SubKey = hash_value(LI); } else if (isVectorLikeInstWithConstOps(V)) { // Sort extracts by the vector operands. if (isa(V)) @@ -4660,7 +4668,7 @@ static std::pair generateKeySubkey( if (isa(I)) { std::pair OpVals = generateKeySubkey(I->getOperand(0), TLI, LoadsSubkeyGenerator, - /*=AllowAlternate*/ true); + /*AllowAlternate=*/true); Key = hash_combine(OpVals.first, Key); SubKey = hash_combine(OpVals.first, SubKey); } @@ -4719,7 +4727,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, &UserTreeIdx, this](const InstructionsState &S) { // Check that every instruction appears once in this bundle. - DenseMap UniquePositions; + DenseMap UniquePositions(VL.size()); for (Value *V : VL) { if (isConstant(V)) { ReuseShuffleIndicies.emplace_back( @@ -4877,7 +4885,7 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, BB && sortPtrAccesses(VL, UserTreeIdx.UserTE->getMainOp()->getType(), *DL, *SE, SortedIndices)); - if (allConstant(VL) || isSplat(VL) || !AreAllSameInsts || + if (!AreAllSameInsts || allConstant(VL) || isSplat(VL) || (isa( S.OpValue) && !all_of(VL, isVectorLikeInstWithConstOps)) || @@ -4951,9 +4959,9 @@ void BoUpSLP::buildTree_rec(ArrayRef VL, unsigned Depth, // Special processing for sorted pointers for ScatterVectorize node with // constant indeces only. - if (AreAllSameInsts && !(S.getOpcode() && allSameBlock(VL)) && - UserTreeIdx.UserTE && - UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize) { + if (AreAllSameInsts && UserTreeIdx.UserTE && + UserTreeIdx.UserTE->State == TreeEntry::ScatterVectorize && + !(S.getOpcode() && allSameBlock(VL))) { assert(S.OpValue->getType()->isPointerTy() && count_if(VL, [](Value *V) { return isa(V); }) >= 2 && @@ -11104,6 +11112,13 @@ class HorizontalReduction { return I->getOperand(getFirstOperandIndex(I) + 1); } + static bool isGoodForReduction(ArrayRef Data) { + int Sz = Data.size(); + auto *I = dyn_cast(Data.front()); + return Sz > 1 || isConstant(Data.front()) || + (I && !isa(I) && isValidForAlternation(I->getOpcode())); + } + public: HorizontalReduction() = default; @@ -11199,6 +11214,9 @@ public: MapVector>> PossibleReducedVals; initReductionOps(Inst); + DenseMap> LoadsMap; + SmallSet LoadKeyUsed; + SmallPtrSet DoNotReverseVals; while (!Worklist.empty()) { Instruction *TreeN = Worklist.pop_back_val(); SmallVector Args; @@ -11220,18 +11238,36 @@ public: size_t Key, Idx; std::tie(Key, Idx) = generateKeySubkey( V, &TLI, - [&PossibleReducedVals, &DL, &SE](size_t Key, LoadInst *LI) { - auto It = PossibleReducedVals.find(Key); - if (It != PossibleReducedVals.end()) { - for (const auto &LoadData : It->second) { - auto *RLI = cast(LoadData.second.front().first); - if (getPointersDiff(RLI->getType(), - RLI->getPointerOperand(), LI->getType(), - LI->getPointerOperand(), DL, SE, - /*StrictCheck=*/true)) - return hash_value(RLI->getPointerOperand()); + [&](size_t Key, LoadInst *LI) { + Value *Ptr = getUnderlyingObject(LI->getPointerOperand()); + if (LoadKeyUsed.contains(Key)) { + auto LIt = LoadsMap.find(Ptr); + if (LIt != LoadsMap.end()) { + for (LoadInst *RLI: LIt->second) { + if (getPointersDiff( + RLI->getType(), RLI->getPointerOperand(), + LI->getType(), LI->getPointerOperand(), DL, SE, + /*StrictCheck=*/true)) + return hash_value(RLI->getPointerOperand()); + } + for (LoadInst *RLI : LIt->second) { + if (arePointersCompatible(RLI->getPointerOperand(), + LI->getPointerOperand())) { + hash_code SubKey = hash_value(RLI->getPointerOperand()); + DoNotReverseVals.insert(RLI); + return SubKey; + } + } + if (LIt->second.size() > 2) { + hash_code SubKey = + hash_value(LIt->second.back()->getPointerOperand()); + DoNotReverseVals.insert(LIt->second.back()); + return SubKey; + } } } + LoadKeyUsed.insert(Key); + LoadsMap.try_emplace(Ptr).first->second.push_back(LI); return hash_value(LI->getPointerOperand()); }, /*AllowAlternate=*/false); @@ -11245,17 +11281,35 @@ public: size_t Key, Idx; std::tie(Key, Idx) = generateKeySubkey( TreeN, &TLI, - [&PossibleReducedVals, &DL, &SE](size_t Key, LoadInst *LI) { - auto It = PossibleReducedVals.find(Key); - if (It != PossibleReducedVals.end()) { - for (const auto &LoadData : It->second) { - auto *RLI = cast(LoadData.second.front().first); - if (getPointersDiff(RLI->getType(), RLI->getPointerOperand(), - LI->getType(), LI->getPointerOperand(), - DL, SE, /*StrictCheck=*/true)) - return hash_value(RLI->getPointerOperand()); + [&](size_t Key, LoadInst *LI) { + Value *Ptr = getUnderlyingObject(LI->getPointerOperand()); + if (LoadKeyUsed.contains(Key)) { + auto LIt = LoadsMap.find(Ptr); + if (LIt != LoadsMap.end()) { + for (LoadInst *RLI: LIt->second) { + if (getPointersDiff(RLI->getType(), + RLI->getPointerOperand(), LI->getType(), + LI->getPointerOperand(), DL, SE, + /*StrictCheck=*/true)) + return hash_value(RLI->getPointerOperand()); + } + for (LoadInst *RLI : LIt->second) { + if (arePointersCompatible(RLI->getPointerOperand(), + LI->getPointerOperand())) { + hash_code SubKey = hash_value(RLI->getPointerOperand()); + DoNotReverseVals.insert(RLI); + return SubKey; + } + } + if (LIt->second.size() > 2) { + hash_code SubKey = hash_value(LIt->second.back()->getPointerOperand()); + DoNotReverseVals.insert(LIt->second.back()); + return SubKey; + } } } + LoadKeyUsed.insert(Key); + LoadsMap.try_emplace(Ptr).first->second.push_back(LI); return hash_value(LI->getPointerOperand()); }, /*AllowAlternate=*/false); @@ -11281,9 +11335,27 @@ public: stable_sort(PossibleRedValsVect, [](const auto &P1, const auto &P2) { return P1.size() > P2.size(); }); - ReducedVals.emplace_back(); - for (ArrayRef Data : PossibleRedValsVect) - ReducedVals.back().append(Data.rbegin(), Data.rend()); + int NewIdx = -1; + for (ArrayRef Data : PossibleRedValsVect) { + if (isGoodForReduction(Data) || + (isa(Data.front()) && NewIdx >= 0 && + isa(ReducedVals[NewIdx].front()) && + getUnderlyingObject( + cast(Data.front())->getPointerOperand()) == + getUnderlyingObject(cast(ReducedVals[NewIdx].front()) + ->getPointerOperand()))) { + if (NewIdx < 0) { + NewIdx = ReducedVals.size(); + ReducedVals.emplace_back(); + } + if (DoNotReverseVals.contains(Data.front())) + ReducedVals[NewIdx].append(Data.begin(), Data.end()); + else + ReducedVals[NewIdx].append(Data.rbegin(), Data.rend()); + } else { + ReducedVals.emplace_back().append(Data.rbegin(), Data.rend()); + } + } } // Sort the reduced values by number of same/alternate opcode and/or pointer // operand. @@ -11301,18 +11373,28 @@ public: // If there are a sufficient number of reduction values, reduce // to a nearby power-of-2. We can safely generate oversized // vectors and rely on the backend to split them to legal sizes. - unsigned NumReducedVals = std::accumulate( - ReducedVals.begin(), ReducedVals.end(), 0, - [](int Num, ArrayRef Vals) { return Num + Vals.size(); }); - if (NumReducedVals < ReductionLimit) + size_t NumReducedVals = + std::accumulate(ReducedVals.begin(), ReducedVals.end(), 0, + [](size_t Num, ArrayRef Vals) { + if (!isGoodForReduction(Vals)) + return Num; + return Num + Vals.size(); + }); + if (NumReducedVals < ReductionLimit) { + for (ReductionOpsType &RdxOps : ReductionOps) + for (Value *RdxOp : RdxOps) + V.analyzedReductionRoot(cast(RdxOp)); return nullptr; + } IRBuilder<> Builder(cast(ReductionRoot)); // Track the reduced values in case if they are replaced by extractelement // because of the vectorization. - DenseMap TrackedVals; + DenseMap TrackedVals( + ReducedVals.size() * ReducedVals.front().size() + ExtraArgs.size()); BoUpSLP::ExtraValueToDebugLocsMap ExternallyUsedValues; + ExternallyUsedValues.reserve(ExtraArgs.size() + 1); // The same extra argument may be used several times, so log each attempt // to use it. for (const std::pair &Pair : ExtraArgs) { @@ -11335,7 +11417,8 @@ public: // The reduction root is used as the insertion point for new instructions, // so set it as externally used to prevent it from being deleted. ExternallyUsedValues[ReductionRoot]; - SmallDenseSet IgnoreList; + SmallDenseSet IgnoreList(ReductionOps.size() * + ReductionOps.front().size()); for (ReductionOpsType &RdxOps : ReductionOps) for (Value *RdxOp : RdxOps) { if (!RdxOp) @@ -11350,7 +11433,7 @@ public: for (Value *V : Candidates) TrackedVals.try_emplace(V, V); - DenseMap VectorizedVals; + DenseMap VectorizedVals(ReducedVals.size()); Value *VectorizedTree = nullptr; bool CheckForReusedReductionOps = false; // Try to vectorize elements based on their type. @@ -11358,7 +11441,8 @@ public: ArrayRef OrigReducedVals = ReducedVals[I]; InstructionsState S = getSameOpcode(OrigReducedVals); SmallVector Candidates; - DenseMap TrackedToOrig; + Candidates.reserve(2 * OrigReducedVals.size()); + DenseMap TrackedToOrig(2 * OrigReducedVals.size()); for (unsigned Cnt = 0, Sz = OrigReducedVals.size(); Cnt < Sz; ++Cnt) { Value *RdxVal = TrackedVals.find(OrigReducedVals[Cnt])->second; // Check if the reduction value was not overriden by the extractelement @@ -11483,18 +11567,14 @@ public: }); } // Number of uses of the candidates in the vector of values. - SmallDenseMap NumUses; + SmallDenseMap NumUses(Candidates.size()); for (unsigned Cnt = 0; Cnt < Pos; ++Cnt) { Value *V = Candidates[Cnt]; - if (NumUses.count(V) > 0) - continue; - NumUses[V] = std::count(VL.begin(), VL.end(), V); + ++NumUses.try_emplace(V, 0).first->getSecond(); } for (unsigned Cnt = Pos + ReduxWidth; Cnt < NumReducedVals; ++Cnt) { Value *V = Candidates[Cnt]; - if (NumUses.count(V) > 0) - continue; - NumUses[V] = std::count(VL.begin(), VL.end(), V); + ++NumUses.try_emplace(V, 0).first->getSecond(); } // Gather externally used values. SmallPtrSet Visited; @@ -11545,9 +11625,8 @@ public: } InstructionCost Cost = TreeCost + ReductionCost; LLVM_DEBUG(dbgs() << "SLP: Found cost = " << Cost << " for reduction\n"); - if (!Cost.isValid()) { + if (!Cost.isValid()) return nullptr; - } if (Cost >= -SLPCostThreshold) { V.getORE()->emit([&]() { return OptimizationRemarkMissed(