f66bd8e81a
This adds a new function-level `flatten` LLVM IR attribute and implements support for it in the AlwaysInliner pass, bringing LLVM's behavior in line with GCC. Previously, the `flatten` attribute only existed as a Clang attribute, which was lowered to `alwaysinline` on individual call sites. Per the RFC discussion [1], the consensus was to match GCC semantics: recursively inline the entire call tree into the flattened function, rather than just immediate call sites. This PR: - Adds the `flatten` function attribute to LLVM IR - Implements recursive inlining of all viable callees in AlwaysInliner - Uses inline history tracking to detect and stop at recursive call cycles - Emits optimization remarks when inlining is skipped due to recursion A follow-up patch will update Clang to emit the LLVM `flatten` attribute on functions instead of marking individual call sites with `alwaysinline`. [1] https://discourse.llvm.org/t/rfc-function-level-flatten-depth-attribute-for-depth-limited-inlining
208 lines
6.5 KiB
LLVM
208 lines
6.5 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes --version 6
|
|
; RUN: opt -passes=always-inline -S < %s | FileCheck %s --check-prefixes=CHECK,ALWAYS
|
|
; RUN: opt -passes=inline -S < %s | FileCheck %s --check-prefixes=CHECK,INLINE
|
|
; RUN: opt -passes='cgscc(inline<only-mandatory>)' -S < %s | FileCheck %s --check-prefixes=CHECK,MANDATORY
|
|
; RUN: opt -passes=always-inline -pass-remarks-missed=inline -S %s -o /dev/null 2>&1 | FileCheck --check-prefix=REMARK %s
|
|
|
|
; Test that the flatten attribute recursively inlines all calls.
|
|
|
|
; Multiple levels are inlined.
|
|
define internal i32 @leaf() {
|
|
; ALWAYS-LABEL: define internal i32 @leaf() {
|
|
; ALWAYS-NEXT: ret i32 42
|
|
;
|
|
ret i32 42
|
|
}
|
|
|
|
define internal i32 @middle() {
|
|
; ALWAYS-LABEL: define internal i32 @middle() {
|
|
; ALWAYS-NEXT: [[R:%.*]] = call i32 @leaf()
|
|
; ALWAYS-NEXT: ret i32 [[R]]
|
|
;
|
|
%r = call i32 @leaf()
|
|
ret i32 %r
|
|
}
|
|
|
|
define i32 @test_multilevel() flatten {
|
|
; CHECK-LABEL: define i32 @test_multilevel(
|
|
; CHECK-SAME: ) #[[ATTR0:[0-9]+]] {
|
|
; CHECK-NEXT: ret i32 42
|
|
;
|
|
%r = call i32 @middle()
|
|
ret i32 %r
|
|
}
|
|
|
|
; Functions with invoke are inlined.
|
|
declare i32 @__gxx_personality_v0(...)
|
|
declare void @may_throw()
|
|
|
|
define internal i32 @callee_with_invoke() personality ptr @__gxx_personality_v0 {
|
|
; ALWAYS-LABEL: define internal i32 @callee_with_invoke() personality ptr @__gxx_personality_v0 {
|
|
; ALWAYS-NEXT: [[ENTRY:.*:]]
|
|
; ALWAYS-NEXT: invoke void @may_throw()
|
|
; ALWAYS-NEXT: to label %[[CONT:.*]] unwind label %[[LPAD:.*]]
|
|
; ALWAYS: [[CONT]]:
|
|
; ALWAYS-NEXT: ret i32 100
|
|
; ALWAYS: [[LPAD]]:
|
|
; ALWAYS-NEXT: [[LP:%.*]] = landingpad { ptr, i32 }
|
|
; ALWAYS-NEXT: cleanup
|
|
; ALWAYS-NEXT: resume { ptr, i32 } [[LP]]
|
|
;
|
|
entry:
|
|
invoke void @may_throw() to label %cont unwind label %lpad
|
|
cont:
|
|
ret i32 100
|
|
lpad:
|
|
%lp = landingpad { ptr, i32 } cleanup
|
|
resume { ptr, i32 } %lp
|
|
}
|
|
|
|
define i32 @test_invoke() flatten personality ptr @__gxx_personality_v0 {
|
|
; CHECK-LABEL: define i32 @test_invoke(
|
|
; CHECK-SAME: ) #[[ATTR0]] personality ptr @__gxx_personality_v0 {
|
|
; CHECK-NEXT: [[ENTRY:.*:]]
|
|
; CHECK-NEXT: invoke void @may_throw()
|
|
; CHECK-NEXT: to label %[[CALLEE_WITH_INVOKE_EXIT:.*]] unwind label %[[LPAD_I:.*]]
|
|
; CHECK: [[LPAD_I]]:
|
|
; CHECK-NEXT: [[LP_I:%.*]] = landingpad { ptr, i32 }
|
|
; CHECK-NEXT: cleanup
|
|
; CHECK-NEXT: resume { ptr, i32 } [[LP_I]]
|
|
; CHECK: [[CALLEE_WITH_INVOKE_EXIT]]:
|
|
; CHECK-NEXT: ret i32 100
|
|
;
|
|
entry:
|
|
%r = call i32 @callee_with_invoke()
|
|
ret i32 %r
|
|
}
|
|
|
|
; Declaration without definition is not inlined.
|
|
declare i32 @external_func()
|
|
|
|
define i32 @test_declaration() flatten {
|
|
; CHECK-LABEL: define i32 @test_declaration(
|
|
; CHECK-SAME: ) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[R:%.*]] = call i32 @external_func()
|
|
; CHECK-NEXT: ret i32 [[R]]
|
|
;
|
|
%r = call i32 @external_func()
|
|
ret i32 %r
|
|
}
|
|
|
|
; Inlined callee that calls a declaration - the declaration should remain after flattening.
|
|
define internal i32 @calls_external() {
|
|
; ALWAYS-LABEL: define internal i32 @calls_external() {
|
|
; ALWAYS-NEXT: [[R:%.*]] = call i32 @external_func()
|
|
; ALWAYS-NEXT: ret i32 [[R]]
|
|
;
|
|
%r = call i32 @external_func()
|
|
ret i32 %r
|
|
}
|
|
|
|
define i32 @test_inline_then_declaration() flatten {
|
|
; CHECK-LABEL: define i32 @test_inline_then_declaration(
|
|
; CHECK-SAME: ) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[R_I:%.*]] = call i32 @external_func()
|
|
; CHECK-NEXT: ret i32 [[R_I]]
|
|
;
|
|
%r = call i32 @calls_external()
|
|
ret i32 %r
|
|
}
|
|
|
|
; Indirect calls are not inlined.
|
|
define internal i32 @target_func() {
|
|
; CHECK-LABEL: define internal i32 @target_func() {
|
|
; CHECK-NEXT: ret i32 99
|
|
;
|
|
ret i32 99
|
|
}
|
|
|
|
define i32 @test_indirect(ptr %func_ptr) flatten {
|
|
; CHECK-LABEL: define i32 @test_indirect(
|
|
; CHECK-SAME: ptr [[FUNC_PTR:%.*]]) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[R:%.*]] = call i32 [[FUNC_PTR]]()
|
|
; CHECK-NEXT: ret i32 [[R]]
|
|
;
|
|
%r = call i32 %func_ptr()
|
|
ret i32 %r
|
|
}
|
|
|
|
; Direct recursion back to flattened function.
|
|
; The callee calls the flattened function - should not cause infinite inlining.
|
|
define internal i32 @calls_flattened_func() {
|
|
; ALWAYS-LABEL: define internal i32 @calls_flattened_func() {
|
|
; ALWAYS-NEXT: [[R:%.*]] = call i32 @test_direct_recursion()
|
|
; ALWAYS-NEXT: ret i32 [[R]]
|
|
;
|
|
%r = call i32 @test_direct_recursion()
|
|
ret i32 %r
|
|
}
|
|
|
|
define i32 @test_direct_recursion() flatten {
|
|
; The call to calls_flattened_func should be inlined, but the recursive call back
|
|
; to test_direct_recursion should remain.
|
|
; CHECK-LABEL: define i32 @test_direct_recursion(
|
|
; CHECK-SAME: ) #[[ATTR0]] {
|
|
; CHECK-NEXT: [[R_I:%.*]] = call i32 @test_direct_recursion()
|
|
; CHECK-NEXT: ret i32 [[R_I]]
|
|
;
|
|
%r = call i32 @calls_flattened_func()
|
|
ret i32 %r
|
|
}
|
|
|
|
; Mutual recursion (A calls B, B calls A).
|
|
; Should inline once but not infinitely.
|
|
define internal i32 @mutual_a() {
|
|
; ALWAYS-LABEL: define internal i32 @mutual_a() {
|
|
; ALWAYS-NEXT: [[R:%.*]] = call i32 @mutual_b()
|
|
; ALWAYS-NEXT: ret i32 [[R]]
|
|
;
|
|
; INLINE-LABEL: define internal i32 @mutual_a() {
|
|
; INLINE-NEXT: [[R_I:%.*]] = call i32 @mutual_a()
|
|
; INLINE-NEXT: ret i32 [[R_I]]
|
|
;
|
|
; MANDATORY-LABEL: define internal i32 @mutual_a() {
|
|
; MANDATORY-NEXT: [[R:%.*]] = call i32 @mutual_b()
|
|
; MANDATORY-NEXT: ret i32 [[R]]
|
|
;
|
|
%r = call i32 @mutual_b()
|
|
ret i32 %r
|
|
}
|
|
|
|
define internal i32 @mutual_b() {
|
|
; ALWAYS-LABEL: define internal i32 @mutual_b() {
|
|
; ALWAYS-NEXT: [[R:%.*]] = call i32 @mutual_a()
|
|
; ALWAYS-NEXT: ret i32 [[R]]
|
|
;
|
|
; MANDATORY-LABEL: define internal i32 @mutual_b() {
|
|
; MANDATORY-NEXT: [[R:%.*]] = call i32 @mutual_a()
|
|
; MANDATORY-NEXT: ret i32 [[R]]
|
|
;
|
|
%r = call i32 @mutual_a()
|
|
ret i32 %r
|
|
}
|
|
|
|
define i32 @test_mutual_recursion() flatten {
|
|
; After inlining mutual_a, we get call to mutual_b.
|
|
; After inlining mutual_b, we get call to mutual_a which should remain (skipped due to inline history).
|
|
; ALWAYS-LABEL: define i32 @test_mutual_recursion(
|
|
; ALWAYS-SAME: ) #[[ATTR0]] {
|
|
; ALWAYS-NEXT: [[R_I1:%.*]] = call i32 @mutual_a()
|
|
; ALWAYS-NEXT: ret i32 [[R_I1]]
|
|
;
|
|
; INLINE-LABEL: define i32 @test_mutual_recursion(
|
|
; INLINE-SAME: ) #[[ATTR0]] {
|
|
; INLINE-NEXT: [[R:%.*]] = call i32 @mutual_a()
|
|
; INLINE-NEXT: ret i32 [[R]]
|
|
;
|
|
; MANDATORY-LABEL: define i32 @test_mutual_recursion(
|
|
; MANDATORY-SAME: ) #[[ATTR0]] {
|
|
; MANDATORY-NEXT: [[R_I1:%.*]] = call i32 @mutual_a()
|
|
; MANDATORY-NEXT: ret i32 [[R_I1]]
|
|
;
|
|
%r = call i32 @mutual_a()
|
|
ret i32 %r
|
|
}
|
|
|
|
; Check that optimization remark is emitted for recursive calls during flattening.
|
|
; REMARK: remark: {{.*}} 'test_direct_recursion' is not inlined into 'test_direct_recursion': recursive call during flattening
|