From 01955119c7740a5e1fdf5bed4ba4434a35fe4613 Mon Sep 17 00:00:00 2001 From: Snehasish Kumar Date: Wed, 11 Mar 2026 09:01:57 -0700 Subject: [PATCH] [InstCombine] Fix profile metadata propagation in InstCombine select folding (#179743) Propagate profile metadata when canonicalizing SPF and drop it when folding select instructions with logical AND/OR conditions. This fixes profile verification failures in Transforms/InstCombine/select-and-or.ll. 1. Select Pattern Factor (SPF) Canonicalization When canonicalizing SPF patterns (like umax/umin), InstCombine transforms sequences like select i1 %cond,(select i1 %cmp, %x, %y), %z into intrinsic calls wrapped in a new select. The new outer select directly replaces the original select instruction, and its condition (%cond) remains structurally identical. Because the condition and its evaluated true/false semantics are unchanged, it is ok to copy the original !prof branch weight metadata to the newly created select. 2. Logical Boolean Folds (foldSelectOfBools) For logical boolean folds (e.g., transforming select (~a | c), a, b into select a, (select c, true, b), false), InstCombine restructures complex conditions into nested selects. In most cases, it is challenging to infer what the profile weights should be for the new select instructions. For a couple of places where should be able to infer profile weights, I've left a TODO to follow up. --- .../InstCombine/InstCombineSelect.cpp | 62 +++++++++++-------- .../Transforms/InstCombine/select-and-or.ll | 41 ++++++------ llvm/utils/profcheck-xfail.txt | 1 - 3 files changed, 58 insertions(+), 46 deletions(-) diff --git a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp index ebdccf88b922..1de03254e818 100644 --- a/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp +++ b/llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp @@ -3595,11 +3595,11 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { if (match(CondVal, m_OneUse(m_Select(m_Value(A), m_One(), m_Value(B)))) && impliesPoisonOrCond(FalseVal, B, /*Expected=*/false)) { // (A || B) || C --> A || (B | C) - return replaceInstUsesWith( - SI, Builder.CreateLogicalOr(A, Builder.CreateOr(B, FalseVal), "", - ProfcheckDisableMetadataFixes - ? nullptr - : cast(CondVal))); + Value *LOr = Builder.CreateLogicalOr(A, Builder.CreateOr(B, FalseVal)); + if (auto *I = dyn_cast(LOr)) { + setExplicitlyUnknownBranchWeightsIfProfiled(*I, DEBUG_TYPE); + } + return replaceInstUsesWith(SI, LOr); } // (A && B) || (C && B) --> (A || C) && B @@ -3611,11 +3611,12 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { auto AndFactorization = [&](Value *Common, Value *InnerCond, Value *InnerVal, bool SelFirst = false) -> Instruction * { - Value *InnerSel = Builder.CreateSelect(InnerCond, One, InnerVal); + Value *InnerSel = Builder.CreateSelectWithUnknownProfile( + InnerCond, One, InnerVal, DEBUG_TYPE); if (SelFirst) std::swap(Common, InnerSel); if (FalseLogicAnd || (CondLogicAnd && Common == A)) - return SelectInst::Create(Common, InnerSel, Zero); + return createSelectInstWithUnknownProfile(Common, InnerSel, Zero); else return BinaryOperator::CreateAnd(Common, InnerSel); }; @@ -3640,11 +3641,11 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { if (match(CondVal, m_OneUse(m_Select(m_Value(A), m_Value(B), m_Zero()))) && impliesPoisonOrCond(TrueVal, B, /*Expected=*/true)) { // (A && B) && C --> A && (B & C) - return replaceInstUsesWith( - SI, Builder.CreateLogicalAnd(A, Builder.CreateAnd(B, TrueVal), "", - ProfcheckDisableMetadataFixes - ? nullptr - : cast(CondVal))); + Value *LAnd = Builder.CreateLogicalAnd(A, Builder.CreateAnd(B, TrueVal)); + if (auto *I = dyn_cast(LAnd)) { + setExplicitlyUnknownBranchWeightsIfProfiled(*I, DEBUG_TYPE); + } + return replaceInstUsesWith(SI, LAnd); } // (A || B) && (C || B) --> (A && C) || B @@ -3656,11 +3657,12 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { auto OrFactorization = [&](Value *Common, Value *InnerCond, Value *InnerVal, bool SelFirst = false) -> Instruction * { - Value *InnerSel = Builder.CreateSelect(InnerCond, InnerVal, Zero); + Value *InnerSel = Builder.CreateSelectWithUnknownProfile( + InnerCond, InnerVal, Zero, DEBUG_TYPE); if (SelFirst) std::swap(Common, InnerSel); if (TrueLogicOr || (CondLogicOr && Common == A)) - return SelectInst::Create(Common, One, InnerSel); + return createSelectInstWithUnknownProfile(Common, One, InnerSel); else return BinaryOperator::CreateOr(Common, InnerSel); }; @@ -3738,28 +3740,36 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) { // select (~a | c), a, b -> select a, (select c, true, b), false if (match(CondVal, m_OneUse(m_c_Or(m_Not(m_Specific(TrueVal)), m_Value(C))))) { - Value *OrV = Builder.CreateSelect(C, One, FalseVal); - return SelectInst::Create(TrueVal, OrV, Zero); + // TODO(#183864): We could improve the profile if P(~a | c) < 0.5, which + // implies strong bounds on both operands (P(a) is high, P(c) is low). + Value *OrV = + Builder.CreateSelectWithUnknownProfile(C, One, FalseVal, DEBUG_TYPE); + return createSelectInstWithUnknownProfile(TrueVal, OrV, Zero); } // select (c & b), a, b -> select b, (select ~c, true, a), false if (match(CondVal, m_OneUse(m_c_And(m_Value(C), m_Specific(FalseVal))))) { if (Value *NotC = getFreelyInverted(C, C->hasOneUse(), &Builder)) { - Value *OrV = Builder.CreateSelect(NotC, One, TrueVal); - return SelectInst::Create(FalseVal, OrV, Zero); + Value *OrV = Builder.CreateSelectWithUnknownProfile(NotC, One, TrueVal, + DEBUG_TYPE); + return createSelectInstWithUnknownProfile(FalseVal, OrV, Zero); } } // select (a | c), a, b -> select a, true, (select ~c, b, false) if (match(CondVal, m_OneUse(m_c_Or(m_Specific(TrueVal), m_Value(C))))) { if (Value *NotC = getFreelyInverted(C, C->hasOneUse(), &Builder)) { - Value *AndV = Builder.CreateSelect(NotC, FalseVal, Zero); - return SelectInst::Create(TrueVal, One, AndV); + // TODO(#183864): We could improve the profile if P(a | c) < 0.5, which + // implies strong bounds on both operands (both P(a) and P(c) are low). + Value *AndV = Builder.CreateSelectWithUnknownProfile(NotC, FalseVal, Zero, + DEBUG_TYPE); + return createSelectInstWithUnknownProfile(TrueVal, One, AndV); } } // select (c & ~b), a, b -> select b, true, (select c, a, false) if (match(CondVal, m_OneUse(m_c_And(m_Value(C), m_Not(m_Specific(FalseVal)))))) { - Value *AndV = Builder.CreateSelect(C, TrueVal, Zero); - return SelectInst::Create(FalseVal, One, AndV); + Value *AndV = + Builder.CreateSelectWithUnknownProfile(C, TrueVal, Zero, DEBUG_TYPE); + return createSelectInstWithUnknownProfile(FalseVal, One, AndV); } if (match(FalseVal, m_Zero()) || match(TrueVal, m_One())) { @@ -4865,9 +4875,11 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) { // Is (select B, T, F) a SPF? if (CondVal->hasOneUse() && SelType->isIntOrIntVectorTy()) { if (ICmpInst *Cmp = dyn_cast(B)) - if (Value *V = canonicalizeSPF(*Cmp, TrueVal, FalseVal, *this)) - return SelectInst::Create(A, IsAnd ? V : TrueVal, - IsAnd ? FalseVal : V); + if (Value *V = canonicalizeSPF(*Cmp, TrueVal, FalseVal, *this)) { + return SelectInst::Create( + A, IsAnd ? V : TrueVal, IsAnd ? FalseVal : V, "", nullptr, + ProfcheckDisableMetadataFixes ? nullptr : &SI); + } } return nullptr; diff --git a/llvm/test/Transforms/InstCombine/select-and-or.ll b/llvm/test/Transforms/InstCombine/select-and-or.ll index 7ae250f04f29..ac1e3f05d608 100644 --- a/llvm/test/Transforms/InstCombine/select-and-or.ll +++ b/llvm/test/Transforms/InstCombine/select-and-or.ll @@ -490,27 +490,27 @@ define i1 @demorgan_select_infloop2(i1 %L) { ret i1 %C15 } -define i1 @and_or1(i1 %a, i1 %b, i1 %c) { +define i1 @and_or1(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @and_or1( -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[B:%.*]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 [[TMP1]], i1 false +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[B:%.*]], !prof [[PROF2:![0-9]+]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 [[TMP1]], i1 false, !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[R]] ; %nota = xor i1 %a, true %cond = or i1 %nota, %c - %r = select i1 %cond, i1 %a, i1 %b + %r = select i1 %cond, i1 %a, i1 %b, !prof !1 ret i1 %r } -define i1 @and_or2(i1 %a, i1 %b, i1 %c) { +define i1 @and_or2(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @and_or2( -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[A:%.*]] -; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 [[TMP1]], i1 false +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[A:%.*]], !prof [[PROF2]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 [[TMP1]], i1 false, !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[R]] ; %notc = xor i1 %c, true %cond = and i1 %notc, %b - %r = select i1 %cond, i1 %a, i1 %b + %r = select i1 %cond, i1 %a, i1 %b, !prof !1 ret i1 %r } @@ -741,27 +741,27 @@ define i1 @and_or3_wrong_operand(i1 %a, i1 %b, i32 %x, i32 %y, i1 %d) { ret i1 %r } -define i1 @or_and1(i1 %a, i1 %b, i1 %c) { +define i1 @or_and1(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @or_and1( -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 false -; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[TMP1]] +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 false, !prof [[PROF2]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[TMP1]], !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[R]] ; %notb = xor i1 %b, true %cond = and i1 %notb, %c - %r = select i1 %cond, i1 %a, i1 %b + %r = select i1 %cond, i1 %a, i1 %b, !prof !1 ret i1 %r } -define i1 @or_and2(i1 %a, i1 %b, i1 %c) { +define i1 @or_and2(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @or_and2( -; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[B:%.*]], i1 false -; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[TMP1]] +; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[B:%.*]], i1 false, !prof [[PROF2]] +; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[TMP1]], !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[R]] ; %notc = xor i1 %c, true %cond = or i1 %notc, %a - %r = select i1 %cond, i1 %a, i1 %b + %r = select i1 %cond, i1 %a, i1 %b, !prof !1 ret i1 %r } @@ -803,7 +803,7 @@ define i1 @fold_or_of_ands_with_select_to_logical1(i1 %a, i1 %b, i1 %c) !prof !0 define i1 @fold_or_of_ands_with_select_to_logical2(i1 %a, i1 %b, i1 %c) !prof !0 { ; CHECK-LABEL: @fold_or_of_ands_with_select_to_logical2( -; CHECK-NEXT: [[OR1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 [[B:%.*]], !prof [[PROF2:![0-9]+]] +; CHECK-NEXT: [[OR1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 [[B:%.*]], !prof [[PROF2]] ; CHECK-NEXT: ret i1 [[OR1]] ; %not = xor i1 %c, true @@ -1117,15 +1117,15 @@ define i1 @or_and3_wrong_operand(i1 %a, i1 %b, i32 %x, i32 %y, i1 %d) { ret i1 %r } -define i8 @test_or_umax(i8 %x, i8 %y, i1 %cond) { +define i8 @test_or_umax(i8 %x, i8 %y, i1 %cond) !prof !0 { ; CHECK-LABEL: @test_or_umax( ; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]]) -; CHECK-NEXT: [[RET:%.*]] = select i1 [[COND:%.*]], i8 [[X]], i8 [[TMP1]] +; CHECK-NEXT: [[RET:%.*]] = select i1 [[COND:%.*]], i8 [[X]], i8 [[TMP1]], !prof [[PROF1]] ; CHECK-NEXT: ret i8 [[RET]] ; %cmp = icmp ugt i8 %x, %y %or = select i1 %cond, i1 true, i1 %cmp - %ret = select i1 %or, i8 %x, i8 %y + %ret = select i1 %or, i8 %x, i8 %y, !prof !2 ret i8 %ret } @@ -1467,6 +1467,7 @@ define i8 @test_logical_commuted_and_ne_a_b(i1 %other_cond, i8 %a, i8 %b) { !0 = !{!"function_entry_count", i64 1000} !1 = !{!"branch_weights", i32 2, i32 3} +!2 = !{!"branch_weights", i32 3, i32 2} ;. ; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) } ; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) } diff --git a/llvm/utils/profcheck-xfail.txt b/llvm/utils/profcheck-xfail.txt index e512a2fa2d79..76c00ce33f80 100644 --- a/llvm/utils/profcheck-xfail.txt +++ b/llvm/utils/profcheck-xfail.txt @@ -115,7 +115,6 @@ Transforms/InstCombine/pow-1.ll Transforms/InstCombine/pow-3.ll Transforms/InstCombine/pow-sqrt.ll Transforms/InstCombine/pull-conditional-binop-through-shift.ll -Transforms/InstCombine/select-and-or.ll Transforms/InstCombine/select-factorize.ll Transforms/InstCombine/select-min-max.ll Transforms/InstCombine/select-of-symmetric-selects.ll