shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

[InstCombine] Fix profile metadata propagation in InstCombine select folding (#179743)

Browse Source 
Propagate profile metadata when canonicalizing SPF and drop it when
folding select instructions with logical AND/OR conditions. This fixes
profile verification failures in Transforms/InstCombine/select-and-or.ll.


 1. Select Pattern Factor (SPF) Canonicalization
When canonicalizing SPF patterns (like umax/umin), InstCombine
transforms sequences like select i1 %cond,(select i1 %cmp, %x, %y), %z
into intrinsic calls wrapped in a new select. The new outer select
directly replaces the original select instruction, and its condition
(%cond) remains structurally identical. Because the condition and its
evaluated true/false semantics are unchanged, it is ok to copy the
original !prof branch weight metadata to the newly created select.

2. Logical Boolean Folds (foldSelectOfBools)
For logical boolean folds (e.g., transforming select (~a | c), a, b into
select a, (select c, true, b), false), InstCombine restructures complex
conditions into nested selects. In most cases, it is challenging to
infer what the profile weights should be for the new select
instructions. For a couple of places where should be able to infer
profile weights, I've left a TODO to follow up.
This commit is contained in:
Snehasish Kumar 2026-03-11 09:01:57 -07:00 committed by GitHub
parent 79d2444dae
commit 01955119c7
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 58 additions and 46 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

62
llvm/lib/Transforms/InstCombine/InstCombineSelect.cpp
View File

@ -3595,11 +3595,11 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
if (match(CondVal, m_OneUse(m_Select(m_Value(A), m_One(), m_Value(B)))) &&
impliesPoisonOrCond(FalseVal, B, /*Expected=*/false)) {
// (A || B) || C --> A || (B | C)
return replaceInstUsesWith(
SI, Builder.CreateLogicalOr(A, Builder.CreateOr(B, FalseVal), "",
ProfcheckDisableMetadataFixes
? nullptr
: cast<SelectInst>(CondVal)));
Value *LOr = Builder.CreateLogicalOr(A, Builder.CreateOr(B, FalseVal));
if (auto *I = dyn_cast<Instruction>(LOr)) {
setExplicitlyUnknownBranchWeightsIfProfiled(*I, DEBUG_TYPE);
}
return replaceInstUsesWith(SI, LOr);
}
// (A && B) || (C && B) --> (A || C) && B
@ -3611,11 +3611,12 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
auto AndFactorization = [&](Value *Common, Value *InnerCond,
Value *InnerVal,
bool SelFirst = false) -> Instruction * {
Value *InnerSel = Builder.CreateSelect(InnerCond, One, InnerVal);
Value *InnerSel = Builder.CreateSelectWithUnknownProfile(
InnerCond, One, InnerVal, DEBUG_TYPE);
if (SelFirst)
std::swap(Common, InnerSel);
if (FalseLogicAnd || (CondLogicAnd && Common == A))
return SelectInst::Create(Common, InnerSel, Zero);
return createSelectInstWithUnknownProfile(Common, InnerSel, Zero);
else
return BinaryOperator::CreateAnd(Common, InnerSel);
};
@ -3640,11 +3641,11 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
if (match(CondVal, m_OneUse(m_Select(m_Value(A), m_Value(B), m_Zero()))) &&
impliesPoisonOrCond(TrueVal, B, /*Expected=*/true)) {
// (A && B) && C --> A && (B & C)
return replaceInstUsesWith(
SI, Builder.CreateLogicalAnd(A, Builder.CreateAnd(B, TrueVal), "",
ProfcheckDisableMetadataFixes
? nullptr
: cast<SelectInst>(CondVal)));
Value *LAnd = Builder.CreateLogicalAnd(A, Builder.CreateAnd(B, TrueVal));
if (auto *I = dyn_cast<Instruction>(LAnd)) {
setExplicitlyUnknownBranchWeightsIfProfiled(*I, DEBUG_TYPE);
}
return replaceInstUsesWith(SI, LAnd);
}
// (A || B) && (C || B) --> (A && C) || B
@ -3656,11 +3657,12 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
auto OrFactorization = [&](Value *Common, Value *InnerCond,
Value *InnerVal,
bool SelFirst = false) -> Instruction * {
Value *InnerSel = Builder.CreateSelect(InnerCond, InnerVal, Zero);
Value *InnerSel = Builder.CreateSelectWithUnknownProfile(
InnerCond, InnerVal, Zero, DEBUG_TYPE);
if (SelFirst)
std::swap(Common, InnerSel);
if (TrueLogicOr || (CondLogicOr && Common == A))
return SelectInst::Create(Common, One, InnerSel);
return createSelectInstWithUnknownProfile(Common, One, InnerSel);
else
return BinaryOperator::CreateOr(Common, InnerSel);
};
@ -3738,28 +3740,36 @@ Instruction *InstCombinerImpl::foldSelectOfBools(SelectInst &SI) {
// select (~a | c), a, b -> select a, (select c, true, b), false
if (match(CondVal,
m_OneUse(m_c_Or(m_Not(m_Specific(TrueVal)), m_Value(C))))) {
Value *OrV = Builder.CreateSelect(C, One, FalseVal);
return SelectInst::Create(TrueVal, OrV, Zero);
// TODO(#183864): We could improve the profile if P(~a | c) < 0.5, which
// implies strong bounds on both operands (P(a) is high, P(c) is low).
Value *OrV =
Builder.CreateSelectWithUnknownProfile(C, One, FalseVal, DEBUG_TYPE);
return createSelectInstWithUnknownProfile(TrueVal, OrV, Zero);
}
// select (c & b), a, b -> select b, (select ~c, true, a), false
if (match(CondVal, m_OneUse(m_c_And(m_Value(C), m_Specific(FalseVal))))) {
if (Value *NotC = getFreelyInverted(C, C->hasOneUse(), &Builder)) {
Value *OrV = Builder.CreateSelect(NotC, One, TrueVal);
return SelectInst::Create(FalseVal, OrV, Zero);
Value *OrV = Builder.CreateSelectWithUnknownProfile(NotC, One, TrueVal,
DEBUG_TYPE);
return createSelectInstWithUnknownProfile(FalseVal, OrV, Zero);
}
}
// select (a | c), a, b -> select a, true, (select ~c, b, false)
if (match(CondVal, m_OneUse(m_c_Or(m_Specific(TrueVal), m_Value(C))))) {
if (Value *NotC = getFreelyInverted(C, C->hasOneUse(), &Builder)) {
Value *AndV = Builder.CreateSelect(NotC, FalseVal, Zero);
return SelectInst::Create(TrueVal, One, AndV);
// TODO(#183864): We could improve the profile if P(a | c) < 0.5, which
// implies strong bounds on both operands (both P(a) and P(c) are low).
Value *AndV = Builder.CreateSelectWithUnknownProfile(NotC, FalseVal, Zero,
DEBUG_TYPE);
return createSelectInstWithUnknownProfile(TrueVal, One, AndV);
}
}
// select (c & ~b), a, b -> select b, true, (select c, a, false)
if (match(CondVal,
m_OneUse(m_c_And(m_Value(C), m_Not(m_Specific(FalseVal)))))) {
Value *AndV = Builder.CreateSelect(C, TrueVal, Zero);
return SelectInst::Create(FalseVal, One, AndV);
Value *AndV =
Builder.CreateSelectWithUnknownProfile(C, TrueVal, Zero, DEBUG_TYPE);
return createSelectInstWithUnknownProfile(FalseVal, One, AndV);
}
if (match(FalseVal, m_Zero()) || match(TrueVal, m_One())) {
@ -4865,9 +4875,11 @@ Instruction *InstCombinerImpl::visitSelectInst(SelectInst &SI) {
// Is (select B, T, F) a SPF?
if (CondVal->hasOneUse() && SelType->isIntOrIntVectorTy()) {
if (ICmpInst *Cmp = dyn_cast<ICmpInst>(B))
if (Value *V = canonicalizeSPF(*Cmp, TrueVal, FalseVal, *this))
return SelectInst::Create(A, IsAnd ? V : TrueVal,
IsAnd ? FalseVal : V);
if (Value *V = canonicalizeSPF(*Cmp, TrueVal, FalseVal, *this)) {
return SelectInst::Create(
A, IsAnd ? V : TrueVal, IsAnd ? FalseVal : V, "", nullptr,
ProfcheckDisableMetadataFixes ? nullptr : &SI);
}
}
return nullptr;

41
llvm/test/Transforms/InstCombine/select-and-or.ll
View File

@ -490,27 +490,27 @@ define i1 @demorgan_select_infloop2(i1 %L) {
ret i1 %C15
}
define i1 @and_or1(i1 %a, i1 %b, i1 %c) {
define i1 @and_or1(i1 %a, i1 %b, i1 %c) !prof !0 {
; CHECK-LABEL: @and_or1(
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[B:%.*]]
; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 [[TMP1]], i1 false
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[B:%.*]], !prof [[PROF2:![0-9]+]]
; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 [[TMP1]], i1 false, !prof [[PROF2]]
; CHECK-NEXT: ret i1 [[R]]
;
%nota = xor i1 %a, true
%cond = or i1 %nota, %c
%r = select i1 %cond, i1 %a, i1 %b
%r = select i1 %cond, i1 %a, i1 %b, !prof !1
ret i1 %r
}
define i1 @and_or2(i1 %a, i1 %b, i1 %c) {
define i1 @and_or2(i1 %a, i1 %b, i1 %c) !prof !0 {
; CHECK-LABEL: @and_or2(
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[A:%.*]]
; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 [[TMP1]], i1 false
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 true, i1 [[A:%.*]], !prof [[PROF2]]
; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 [[TMP1]], i1 false, !prof [[PROF2]]
; CHECK-NEXT: ret i1 [[R]]
;
%notc = xor i1 %c, true
%cond = and i1 %notc, %b
%r = select i1 %cond, i1 %a, i1 %b
%r = select i1 %cond, i1 %a, i1 %b, !prof !1
ret i1 %r
}
@ -741,27 +741,27 @@ define i1 @and_or3_wrong_operand(i1 %a, i1 %b, i32 %x, i32 %y, i1 %d) {
ret i1 %r
}
define i1 @or_and1(i1 %a, i1 %b, i1 %c) {
define i1 @or_and1(i1 %a, i1 %b, i1 %c) !prof !0 {
; CHECK-LABEL: @or_and1(
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 false
; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[TMP1]]
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 false, !prof [[PROF2]]
; CHECK-NEXT: [[R:%.*]] = select i1 [[B:%.*]], i1 true, i1 [[TMP1]], !prof [[PROF2]]
; CHECK-NEXT: ret i1 [[R]]
;
%notb = xor i1 %b, true
%cond = and i1 %notb, %c
%r = select i1 %cond, i1 %a, i1 %b
%r = select i1 %cond, i1 %a, i1 %b, !prof !1
ret i1 %r
}
define i1 @or_and2(i1 %a, i1 %b, i1 %c) {
define i1 @or_and2(i1 %a, i1 %b, i1 %c) !prof !0 {
; CHECK-LABEL: @or_and2(
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[B:%.*]], i1 false
; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[TMP1]]
; CHECK-NEXT: [[TMP1:%.*]] = select i1 [[C:%.*]], i1 [[B:%.*]], i1 false, !prof [[PROF2]]
; CHECK-NEXT: [[R:%.*]] = select i1 [[A:%.*]], i1 true, i1 [[TMP1]], !prof [[PROF2]]
; CHECK-NEXT: ret i1 [[R]]
;
%notc = xor i1 %c, true
%cond = or i1 %notc, %a
%r = select i1 %cond, i1 %a, i1 %b
%r = select i1 %cond, i1 %a, i1 %b, !prof !1
ret i1 %r
}
@ -803,7 +803,7 @@ define i1 @fold_or_of_ands_with_select_to_logical1(i1 %a, i1 %b, i1 %c) !prof !0
define i1 @fold_or_of_ands_with_select_to_logical2(i1 %a, i1 %b, i1 %c) !prof !0 {
; CHECK-LABEL: @fold_or_of_ands_with_select_to_logical2(
; CHECK-NEXT: [[OR1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 [[B:%.*]], !prof [[PROF2:![0-9]+]]
; CHECK-NEXT: [[OR1:%.*]] = select i1 [[C:%.*]], i1 [[A:%.*]], i1 [[B:%.*]], !prof [[PROF2]]
; CHECK-NEXT: ret i1 [[OR1]]
;
%not = xor i1 %c, true
@ -1117,15 +1117,15 @@ define i1 @or_and3_wrong_operand(i1 %a, i1 %b, i32 %x, i32 %y, i1 %d) {
ret i1 %r
}
define i8 @test_or_umax(i8 %x, i8 %y, i1 %cond) {
define i8 @test_or_umax(i8 %x, i8 %y, i1 %cond) !prof !0 {
; CHECK-LABEL: @test_or_umax(
; CHECK-NEXT: [[TMP1:%.*]] = call i8 @llvm.umax.i8(i8 [[X:%.*]], i8 [[Y:%.*]])
; CHECK-NEXT: [[RET:%.*]] = select i1 [[COND:%.*]], i8 [[X]], i8 [[TMP1]]
; CHECK-NEXT: [[RET:%.*]] = select i1 [[COND:%.*]], i8 [[X]], i8 [[TMP1]], !prof [[PROF1]]
; CHECK-NEXT: ret i8 [[RET]]
;
%cmp = icmp ugt i8 %x, %y
%or = select i1 %cond, i1 true, i1 %cmp
%ret = select i1 %or, i8 %x, i8 %y
%ret = select i1 %or, i8 %x, i8 %y, !prof !2
ret i8 %ret
}
@ -1467,6 +1467,7 @@ define i8 @test_logical_commuted_and_ne_a_b(i1 %other_cond, i8 %a, i8 %b) {
!0 = !{!"function_entry_count", i64 1000}
!1 = !{!"branch_weights", i32 2, i32 3}
!2 = !{!"branch_weights", i32 3, i32 2}
;.
; CHECK: attributes #[[ATTR0:[0-9]+]] = { nocallback nocreateundeforpoison nofree nosync nounwind speculatable willreturn memory(none) }
; CHECK: attributes #[[ATTR1:[0-9]+]] = { nocallback nofree nosync nounwind speculatable willreturn memory(none) }

1
llvm/utils/profcheck-xfail.txt
View File

@ -115,7 +115,6 @@ Transforms/InstCombine/pow-1.ll
Transforms/InstCombine/pow-3.ll
Transforms/InstCombine/pow-sqrt.ll
Transforms/InstCombine/pull-conditional-binop-through-shift.ll
Transforms/InstCombine/select-and-or.ll
Transforms/InstCombine/select-factorize.ll
Transforms/InstCombine/select-min-max.ll
Transforms/InstCombine/select-of-symmetric-selects.ll