diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index ebdccf88b922..1de03254e818 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -3595,11 +3595,11 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { if (match(CondVal, m_OneUse(m_Select(m_Value(A), m_One(), m_Value(B)))) && impliesPoisonOrCond(FalseVal, B, /*Expected=*/false)) { // (A || B) || C --> A || (B | C) - return replaceInstUsesWith( - SI, Builder.CreateLogicalOr(A, Builder.CreateOr(B, FalseVal), "", - ProfcheckDisableMetadataFixes - ? nullptr - : cast(CondVal))); + Value *LOr = Builder.CreateLogicalOr(A, Builder.CreateOr(B, FalseVal)); + if (auto *I = dyn_cast(LOr)) { + setExplicitlyUnknownBranchWeightsIfProfiled(*I, DEBUG_TYPE); + } + return replaceInstUsesWith(SI, LOr); } // (A && B) || (C && B) --> (A || C) && B @@ -3611,11 +3611,12 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { auto AndFactorization = [&](Value *Common, Value *InnerCond, Value *InnerVal, bool SelFirst = false) -> Instruction * { - Value *InnerSel = Builder.CreateSelect(InnerCond, One, InnerVal); + Value *InnerSel = Builder.CreateSelectWithUnknownProfile( + InnerCond, One, InnerVal, DEBUG_TYPE); if (SelFirst) std::swap(Common, InnerSel); if (FalseLogicAnd || (CondLogicAnd && Common == A)) - return SelectInst::Create(Common, InnerSel, Zero); + return createSelectInstWithUnknownProfile(Common, InnerSel, Zero); else return BinaryOperator::CreateAnd(Common, InnerSel); }; @@ -3640,11 +3641,11 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { if (match(CondVal, m_OneUse(m_Select(m_Value(A), m_Value(B), m_Zero()))) && impliesPoisonOrCond(TrueVal, B, /*Expected=*/true)) { // (A && B) && C --> A && (B & C) - return replaceInstUsesWith( - SI, Builder.CreateLogicalAnd(A, Builder.CreateAnd(B, TrueVal), "", - ProfcheckDisableMetadataFixes - ? nullptr - : cast(CondVal))); + Value *LAnd = Builder.CreateLogicalAnd(A, Builder.CreateAnd(B, TrueVal)); + if (auto *I = dyn_cast(LAnd)) { + setExplicitlyUnknownBranchWeightsIfProfiled(*I, DEBUG_TYPE); + } + return replaceInstUsesWith(SI, LAnd); } // (A || B) && (C || B) --> (A && C) || B @@ -3656,11 +3657,12 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { auto OrFactorization = [&](Value *Common, Value *InnerCond, Value *InnerVal, bool SelFirst = false) -> Instruction * { - Value *InnerSel = Builder.CreateSelect(InnerCond, InnerVal, Zero); + Value *InnerSel = Builder.CreateSelectWithUnknownProfile( + InnerCond, InnerVal, Zero, DEBUG_TYPE); if (SelFirst) std::swap(Common, InnerSel); if (TrueLogicOr || (CondLogicOr && Common == A)) - return SelectInst::Create(Common, One, InnerSel); + return createSelectInstWithUnknownProfile(Common, One, InnerSel); else return BinaryOperator::CreateOr(Common, InnerSel); }; @@ -3738,28 +3740,36 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { // select (~a | c), a, b -> select a, (select c, true, b), false if (match(CondVal, m_OneUse(m_c_Or(m_Not(m_Specific(TrueVal)), m_Value(C))))) { - Value *OrV = Builder.CreateSelect(C, One, FalseVal); - return SelectInst::Create(TrueVal, OrV, Zero); + // TODO(#183864): We could improve the profile if P(~a | c) < 0.5, which + // implies strong bounds on both operands (P(a) is high, P(c) is low). + Value *OrV = + Builder.CreateSelectWithUnknownProfile(C, One, FalseVal, DEBUG_TYPE); + return createSelectInstWithUnknownProfile(TrueVal, OrV, Zero); } // select (c & b), a, b -> select b, (select ~c, true, a), false if (match(CondVal, m_OneUse(m_c_And(m_Value(C), m_Specific(FalseVal))))) { if (Value *NotC = getFreelyInverted(C, C->hasOneUse(), &Builder)) { - Value *OrV = Builder.CreateSelect(NotC, One, TrueVal); - return SelectInst::Create(FalseVal, OrV, Zero); + Value *OrV = Builder.CreateSelectWithUnknownProfile(NotC, One, TrueVal, + DEBUG_TYPE); + return createSelectInstWithUnknownProfile(FalseVal, OrV, Zero); } } // select (a | c), a, b -> select a, true, (select ~c, b, false) if (match(CondVal, m_OneUse(m_c_Or(m_Specific(TrueVal), m_Value(C))))) { if (Value *NotC = getFreelyInverted(C, C->hasOneUse(), &Builder)) { - Value *AndV = Builder.CreateSelect(NotC, FalseVal, Zero); - return SelectInst::Create(TrueVal, One, AndV); + // TODO(#183864): We could improve the profile if P(a | c) < 0.5, which + // implies strong bounds on both operands (both P(a) and P(c) are low). + Value *AndV = Builder.CreateSelectWithUnknownProfile(NotC, FalseVal, Zero, + DEBUG_TYPE); + return createSelectInstWithUnknownProfile(TrueVal, One, AndV); } } // select (c & ~b), a, b -> select b, true, (select c, a, false) if (match(CondVal, m_OneUse(m_c_And(m_Value(C), m_Not(m_Specific(FalseVal)))))) { - Value *AndV = Builder.CreateSelect(C, TrueVal, Zero); - return SelectInst::Create(FalseVal, One, AndV); + Value *AndV = + Builder.CreateSelectWithUnknownProfile(C, TrueVal, Zero, DEBUG_TYPE); + return createSelectInstWithUnknownProfile(FalseVal, One, AndV); } if (match(FalseVal, m_Zero()) || match(TrueVal, m_One())) { @@ -4865,9 +4875,11 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { // Is (select B, T, F) a SPF? if (CondVal->hasOneUse() && SelType->isIntOrIntVectorTy()) { if (ICmpInst *Cmp = dyn_cast(B)) - if (Value *V = canonicalizeSPF(*Cmp, TrueVal, FalseVal, *this)) - return SelectInst::Create(A, IsAnd ? V : TrueVal, - IsAnd ? FalseVal : V); + if (Value *V = canonicalizeSPF(*Cmp, TrueVal, FalseVal, *this)) { + return SelectInst::Create( + A, IsAnd ? V : TrueVal, IsAnd ? FalseVal : V, "", nullptr, + ProfcheckDisableMetadataFixes ? nullptr : &SI); + } } return nullptr; diff --git a/llvm/test/Transforms/InstCombine/select-and-or.ll b/llvm/test/Transforms/InstCombine/select-and-or.ll index 7ae250f04f29..ac1e3f05d608 100644 --- a/llvm/test/Transforms/InstCombine/select-and-or.ll +++ b/llvm/test/Transforms/InstCombine/select-and-or.ll @@ -490,27 +490,27 @@ define i1 @demorgan_select_infloop2(i1 %L) { ret i1 %C15 } -define i1 @and_or1(i1 %a, i1 %b, i1 %c) { +define i1 @and_or1(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @and_or1( -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[B:%.*]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 [[TMP1]], i1 false +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[B:%.*]], !prof [[PROF2:![0-9]+]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 [[TMP1]], i1 false, !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[R]] ; %nota = xor i1 %a, true %cond = or i1 %nota, %c - %r = select i1 %cond, i1 %a, i1 %b + %r = select i1 %cond, i1 %a, i1 %b, !prof !1 ret i1 %r } -define i1 @and_or2(i1 %a, i1 %b, i1 %c) { +define i1 @and_or2(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @and_or2( -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[A:%.*]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 [[TMP1]], i1 false +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[A:%.*]], !prof [[PROF2]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 [[TMP1]], i1 false, !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[R]] ; %notc = xor i1 %c, true %cond = and i1 %notc, %b - %r = select i1 %cond, i1 %a, i1 %b + %r = select i1 %cond, i1 %a, i1 %b, !prof !1 ret i1 %r } @@ -741,27 +741,27 @@ define i1 @and_or3_wrong_operand(i1 %a, i1 %b, i32 %x, i32 %y, i1 %d) { ret i1 %r } -define i1 @or_and1(i1 %a, i1 %b, i1 %c) { +define i1 @or_and1(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @or_and1( -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 false -; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[TMP1]] +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 false, !prof [[PROF2]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[TMP1]], !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[R]] ; %notb = xor i1 %b, true %cond = and i1 %notb, %c - %r = select i1 %cond, i1 %a, i1 %b + %r = select i1 %cond, i1 %a, i1 %b, !prof !1 ret i1 %r } -define i1 @or_and2(i1 %a, i1 %b, i1 %c) { +define i1 @or_and2(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @or_and2( -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[B:%.*]], i1 false -; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[TMP1]] +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[B:%.*]], i1 false, !prof [[PROF2]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[TMP1]], !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[R]] ; %notc = xor i1 %c, true %cond = or i1 %notc, %a - %r = select i1 %cond, i1 %a, i1 %b + %r = select i1 %cond, i1 %a, i1 %b, !prof !1 ret i1 %r } @@ -803,7 +803,7 @@ define i1 @fold_or_of_ands_with_select_to_logical1(i1 %a, i1 %b, i1 %c) !prof !0 define i1 @fold_or_of_ands_with_select_to_logical2(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @fold_or_of_ands_with_select_to_logical2( -; CHECK-NEXT: [[OR1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 [[B:%.*]], !prof [[PROF2:![0-9]+]] +; CHECK-NEXT: [[OR1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 [[B:%.*]], !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[OR1]] ; %not = xor i1 %c, true @@ -1117,15 +1117,15 @@ define i1 @or_and3_wrong_operand(i1 %a, i1 %b, i32 %x, i32 %y, i1 %d) { ret i1 %r } -define i8 @test_or_umax(i8 %x, i8 %y, i1 %cond) { +define i8 @test_or_umax(i8 %x, i8 %y, i1 %cond) !prof !0 { ; CHECK-LABEL: @test_or_umax( ; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]]) -; CHECK-NEXT: [[RET:%.*]] = select i1 [[COND:%.*]], i8 [[X]], i8 [[TMP1]] +; CHECK-NEXT: [[RET:%.*]] = select i1 [[COND:%.*]], i8 [[X]], i8 [[TMP1]], !prof [[PROF1]] ; CHECK-NEXT: ret i8 [[RET]] ; %cmp = icmp ugt i8 %x, %y %or = select i1 %cond, i1 true, i1 %cmp - %ret = select i1 %or, i8 %x, i8 %y + %ret = select i1 %or, i8 %x, i8 %y, !prof !2 ret i8 %ret } @@ -1467,6 +1467,7 @@ define i8 @test_logical_commuted_and_ne_a_b(i1 %other_cond, i8 %a, i8 %b) { !0 = !{!"function_entry_count", i64 1000} !1 = !{!"branch_weights", i32 2, i32 3} +!2 = !{!"branch_weights", i32 3, i32 2} ;. ; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) } ; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt index e512a2fa2d79..76c00ce33f80 100644 --- a/llvm/utils/profcheck-xfail.txt +++ b/llvm/utils/profcheck-xfail.txt @@ -115,7 +115,6 @@ Transforms/InstCombine/pow-1.ll Transforms/InstCombine/pow-3.ll Transforms/InstCombine/pow-sqrt.ll Transforms/InstCombine/pull-conditional-binop-through-shift.ll -Transforms/InstCombine/select-and-or.ll Transforms/InstCombine/select-factorize.ll Transforms/InstCombine/select-min-max.ll Transforms/InstCombine/select-of-symmetric-selects.ll