diff --git a/llvm/include/llvm/Bitcode/LLVMBitCodes.h b/llvm/include/llvm/Bitcode/LLVMBitCodes.h index d9614bce5450..9162754bbfe1 100644 --- a/llvm/include/llvm/Bitcode/LLVMBitCodes.h +++ b/llvm/include/llvm/Bitcode/LLVMBitCodes.h @@ -815,6 +815,7 @@ enum AttributeKindCodes { ATTR_KIND_NO_CREATE_UNDEF_OR_POISON = 105, ATTR_KIND_DENORMAL_FPENV = 106, ATTR_KIND_NOOUTLINE = 107, + ATTR_KIND_FLATTEN = 108, }; enum ComdatSelectionKindCodes { diff --git a/llvm/include/llvm/IR/Attributes.td b/llvm/include/llvm/IR/Attributes.td index 88b30c7e3f71..4bc975a8ef21 100644 --- a/llvm/include/llvm/IR/Attributes.td +++ b/llvm/include/llvm/IR/Attributes.td @@ -134,6 +134,9 @@ def DisableSanitizerInstrumentation: EnumAttr<"disable_sanitizer_instrumentation /// Provide pointer element type to intrinsic. def ElementType : TypeAttr<"elementtype", IntersectPreserve, [ParamAttr]>; +/// Flatten function by recursively inlining all calls. +def Flatten : EnumAttr<"flatten", IntersectPreserve, [FnAttr]>; + /// Whether to keep return instructions, or replace with a jump to an external /// symbol. def FnRetThunkExtern : EnumAttr<"fn_ret_thunk_extern", IntersectPreserve, [FnAttr]>; diff --git a/llvm/include/llvm/Transforms/Utils/Cloning.h b/llvm/include/llvm/Transforms/Utils/Cloning.h index 4f594bb45635..ff4b390eb3e6 100644 --- a/llvm/include/llvm/Transforms/Utils/Cloning.h +++ b/llvm/include/llvm/Transforms/Utils/Cloning.h @@ -17,6 +17,7 @@ #ifndef LLVM_TRANSFORMS_UTILS_CLONING_H #define LLVM_TRANSFORMS_UTILS_CLONING_H +#include "llvm/ADT/ArrayRef.h" #include "llvm/ADT/SmallVector.h" #include "llvm/ADT/Twine.h" #include "llvm/Analysis/AssumptionCache.h" @@ -432,6 +433,14 @@ LLVM_ABI void cloneAndAdaptNoAliasScopes(ArrayRef NoAliasDeclScopes, LLVM_ABI void cloneAndAdaptNoAliasScopes(ArrayRef NoAliasDeclScopes, Instruction *IStart, Instruction *IEnd, LLVMContext &Context, StringRef Ext); +/// Check if Function F appears in the inline history chain. +/// InlineHistory is a vector of (Function, ParentHistoryID) pairs. +/// Returns true if F was already inlined in the chain leading to +/// InlineHistoryID. +LLVM_ABI bool +inlineHistoryIncludes(Function *F, int InlineHistoryID, + ArrayRef> InlineHistory); + } // end namespace llvm #endif // LLVM_TRANSFORMS_UTILS_CLONING_H diff --git a/llvm/lib/Analysis/InlineCost.cpp b/llvm/lib/Analysis/InlineCost.cpp index 06ff9cc80f63..d674743669ba 100644 --- a/llvm/lib/Analysis/InlineCost.cpp +++ b/llvm/lib/Analysis/InlineCost.cpp @@ -3229,6 +3229,15 @@ std::optional llvm::getAttributeBasedInliningDecision( if (!functionsHaveCompatibleAttributes(Caller, Callee, CalleeTTI, GetTLI)) return InlineResult::failure("conflicting attributes"); + // Flatten: inline all viable calls from flatten functions regardless of cost. + // Checked before optnone so that flatten takes priority. + if (Caller->hasFnAttribute(Attribute::Flatten)) { + auto IsViable = isInlineViable(*Callee); + if (IsViable.isSuccess()) + return InlineResult::success(); + return InlineResult::failure(IsViable.getFailureReason()); + } + // Don't inline this call if the caller has the optnone attribute. if (Caller->hasOptNone()) return InlineResult::failure("optnone attribute"); diff --git a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp index 073c1f591608..e01cf501b884 100644 --- a/llvm/lib/Bitcode/Reader/BitcodeReader.cpp +++ b/llvm/lib/Bitcode/Reader/BitcodeReader.cpp @@ -2113,6 +2113,8 @@ static Attribute::AttrKind getAttrFromCode(uint64_t Code) { return Attribute::ElementType; case bitc::ATTR_KIND_FNRETTHUNK_EXTERN: return Attribute::FnRetThunkExtern; + case bitc::ATTR_KIND_FLATTEN: + return Attribute::Flatten; case bitc::ATTR_KIND_INLINE_HINT: return Attribute::InlineHint; case bitc::ATTR_KIND_IN_REG: diff --git a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp index cdc6b2af9501..71d5a8bc98a4 100644 --- a/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp +++ b/llvm/lib/Bitcode/Writer/BitcodeWriter.cpp @@ -779,6 +779,8 @@ static uint64_t getAttrKindEncoding(Attribute::AttrKind Kind) { return bitc::ATTR_KIND_DISABLE_SANITIZER_INSTRUMENTATION; case Attribute::FnRetThunkExtern: return bitc::ATTR_KIND_FNRETTHUNK_EXTERN; + case Attribute::Flatten: + return bitc::ATTR_KIND_FLATTEN; case Attribute::Hot: return bitc::ATTR_KIND_HOT; case Attribute::ElementType: diff --git a/llvm/lib/Transforms/IPO/AlwaysInliner.cpp b/llvm/lib/Transforms/IPO/AlwaysInliner.cpp index 4fba4475767f..d0e309427d32 100644 --- a/llvm/lib/Transforms/IPO/AlwaysInliner.cpp +++ b/llvm/lib/Transforms/IPO/AlwaysInliner.cpp @@ -19,6 +19,8 @@ #include "llvm/Analysis/InlineCost.h" #include "llvm/Analysis/OptimizationRemarkEmitter.h" #include "llvm/Analysis/ProfileSummaryInfo.h" +#include "llvm/Analysis/TargetLibraryInfo.h" +#include "llvm/Analysis/TargetTransformInfo.h" #include "llvm/IR/Module.h" #include "llvm/InitializePasses.h" #include "llvm/Transforms/Utils/Cloning.h" @@ -34,12 +36,49 @@ bool AlwaysInlineImpl( Module &M, bool InsertLifetime, ProfileSummaryInfo &PSI, FunctionAnalysisManager *FAM, function_ref GetAssumptionCache, - function_ref GetAAR) { + function_ref GetAAR, + function_ref GetTTI, + function_ref GetTLI) { SmallSetVector Calls; bool Changed = false; SmallVector InlinedComdatFunctions; + SmallVector NeedFlattening; + + auto TryInline = [&](CallBase &CB, Function &Callee, + OptimizationRemarkEmitter &ORE, const char *InlineReason, + SmallVectorImpl *NewCallSites = + nullptr) -> bool { + Function *Caller = CB.getCaller(); + DebugLoc DLoc = CB.getDebugLoc(); + BasicBlock *Block = CB.getParent(); + + InlineFunctionInfo IFI(GetAssumptionCache, &PSI); + InlineResult Res = InlineFunction(CB, IFI, /*MergeAttributes=*/true, + &GetAAR(Callee), InsertLifetime); + if (!Res.isSuccess()) { + ORE.emit([&]() { + return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) + << "'" << ore::NV("Callee", &Callee) << "' is not inlined into '" + << ore::NV("Caller", Caller) + << "': " << ore::NV("Reason", Res.getFailureReason()); + }); + return false; + } + + emitInlinedIntoBasedOnCost(ORE, DLoc, Block, Callee, *Caller, + InlineCost::getAlways(InlineReason), + /*ForProfileContext=*/false, DEBUG_TYPE); + if (FAM) + FAM->invalidate(*Caller, PreservedAnalyses::none()); + if (NewCallSites) + *NewCallSites = std::move(IFI.InlinedCallSites); + return true; + }; for (Function &F : make_early_inc_range(M)) { + if (F.hasFnAttribute(Attribute::Flatten)) + NeedFlattening.push_back(&F); + if (F.isPresplitCoroutine()) continue; @@ -56,38 +95,12 @@ bool AlwaysInlineImpl( Calls.insert(CB); for (CallBase *CB : Calls) { - Function *Caller = CB->getCaller(); - OptimizationRemarkEmitter ORE(Caller); - DebugLoc DLoc = CB->getDebugLoc(); - BasicBlock *Block = CB->getParent(); - - InlineFunctionInfo IFI(GetAssumptionCache, &PSI, nullptr, nullptr); - InlineResult Res = InlineFunction(*CB, IFI, /*MergeAttributes=*/true, - &GetAAR(F), InsertLifetime); - if (!Res.isSuccess()) { - ORE.emit([&]() { - return OptimizationRemarkMissed(DEBUG_TYPE, "NotInlined", DLoc, Block) - << "'" << ore::NV("Callee", &F) << "' is not inlined into '" - << ore::NV("Caller", Caller) - << "': " << ore::NV("Reason", Res.getFailureReason()); - }); - continue; - } - - emitInlinedIntoBasedOnCost( - ORE, DLoc, Block, F, *Caller, - InlineCost::getAlways("always inline attribute"), - /*ForProfileContext=*/false, DEBUG_TYPE); - - Changed = true; - if (FAM) - FAM->invalidate(*Caller, PreservedAnalyses::none()); + OptimizationRemarkEmitter ORE(CB->getCaller()); + Changed |= TryInline(*CB, F, ORE, "always inline attribute"); } F.removeDeadConstantUsers(); if (F.hasFnAttribute(Attribute::AlwaysInline) && F.isDefTriviallyDead()) { - // Remember to try and delete this function afterward. This allows to call - // filterDeadComdatFunctions() only once. if (F.hasComdat()) { InlinedComdatFunctions.push_back(&F); } else { @@ -99,6 +112,73 @@ bool AlwaysInlineImpl( } } + // Flatten functions with the flatten attribute using a local worklist. + for (Function *F : NeedFlattening) { + SmallVector, 16> Worklist; + SmallVector, 16> InlineHistory; + SmallVector NewCallSites; + OptimizationRemarkEmitter ORE(F); + + // Collect initial calls. + for (BasicBlock &BB : *F) { + for (Instruction &I : BB) { + if (auto *CB = dyn_cast(&I)) { + Function *Callee = CB->getCalledFunction(); + if (!Callee || Callee->isDeclaration()) + continue; + Worklist.push_back({CB, -1}); + } + } + } + + while (!Worklist.empty()) { + auto Item = Worklist.pop_back_val(); + CallBase *CB = Item.first; + int InlineHistoryID = Item.second; + Function *Callee = CB->getCalledFunction(); + if (!Callee) + continue; + + // Detect recursion. + if (Callee == F || + inlineHistoryIncludes(Callee, InlineHistoryID, InlineHistory)) { + ORE.emit([&]() { + return OptimizationRemarkMissed("inline", "NotInlined", + CB->getDebugLoc(), CB->getParent()) + << "'" << ore::NV("Callee", Callee) + << "' is not inlined into '" + << ore::NV("Caller", CB->getCaller()) + << "': recursive call during flattening"; + }); + continue; + } + + // Use getAttributeBasedInliningDecision for all attribute-based checks + // including TTI/TLI compatibility and isInlineViable. + TargetTransformInfo &CalleeTTI = GetTTI(*Callee); + auto Decision = + getAttributeBasedInliningDecision(*CB, Callee, CalleeTTI, GetTLI); + if (!Decision || !Decision->isSuccess()) + continue; + + if (!TryInline(*CB, *Callee, ORE, "flatten attribute", &NewCallSites)) + continue; + + Changed = true; + + // Add new call sites from the inlined function to the worklist. + if (!NewCallSites.empty()) { + int NewHistoryID = InlineHistory.size(); + InlineHistory.push_back({Callee, InlineHistoryID}); + for (CallBase *NewCB : NewCallSites) { + Function *NewCallee = NewCB->getCalledFunction(); + if (NewCallee && !NewCallee->isDeclaration()) + Worklist.push_back({NewCB, NewHistoryID}); + } + } + } + } + if (!InlinedComdatFunctions.empty()) { // Now we just have the comdat functions. Filter out the ones whose comdats // are not actually dead. @@ -134,9 +214,15 @@ struct AlwaysInlinerLegacyPass : public ModulePass { auto GetAssumptionCache = [&](Function &F) -> AssumptionCache & { return getAnalysis().getAssumptionCache(F); }; + auto GetTTI = [&](Function &F) -> TargetTransformInfo & { + return getAnalysis().getTTI(F); + }; + auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { + return getAnalysis().getTLI(F); + }; return AlwaysInlineImpl(M, InsertLifetime, PSI, /*FAM=*/nullptr, - GetAssumptionCache, GetAAR); + GetAssumptionCache, GetAAR, GetTTI, GetTLI); } static char ID; // Pass identification, replacement for typeid @@ -145,6 +231,8 @@ struct AlwaysInlinerLegacyPass : public ModulePass { AU.addRequired(); AU.addRequired(); AU.addRequired(); + AU.addRequired(); + AU.addRequired(); } }; @@ -156,6 +244,8 @@ INITIALIZE_PASS_BEGIN(AlwaysInlinerLegacyPass, "always-inline", INITIALIZE_PASS_DEPENDENCY(AAResultsWrapperPass) INITIALIZE_PASS_DEPENDENCY(AssumptionCacheTracker) INITIALIZE_PASS_DEPENDENCY(ProfileSummaryInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetLibraryInfoWrapperPass) +INITIALIZE_PASS_DEPENDENCY(TargetTransformInfoWrapperPass) INITIALIZE_PASS_END(AlwaysInlinerLegacyPass, "always-inline", "Inliner for always_inline functions", false, false) @@ -173,10 +263,16 @@ PreservedAnalyses AlwaysInlinerPass::run(Module &M, auto GetAAR = [&](Function &F) -> AAResults & { return FAM.getResult(F); }; + auto GetTTI = [&](Function &F) -> TargetTransformInfo & { + return FAM.getResult(F); + }; + auto GetTLI = [&](Function &F) -> const TargetLibraryInfo & { + return FAM.getResult(F); + }; auto &PSI = MAM.getResult(M); bool Changed = AlwaysInlineImpl(M, InsertLifetime, PSI, &FAM, - GetAssumptionCache, GetAAR); + GetAssumptionCache, GetAAR, GetTTI, GetTLI); if (!Changed) return PreservedAnalyses::all(); diff --git a/llvm/lib/Transforms/IPO/Inliner.cpp b/llvm/lib/Transforms/IPO/Inliner.cpp index fb376562f678..d795fbbbe412 100644 --- a/llvm/lib/Transforms/IPO/Inliner.cpp +++ b/llvm/lib/Transforms/IPO/Inliner.cpp @@ -142,21 +142,6 @@ static cl::opt CGSCCInlineReplayFormat( ":. (default)")), cl::desc("How cgscc inline replay file is formatted"), cl::Hidden); -/// Return true if the specified inline history ID -/// indicates an inline history that includes the specified function. -static bool inlineHistoryIncludes( - Function *F, int InlineHistoryID, - const SmallVectorImpl> &InlineHistory) { - while (InlineHistoryID != -1) { - assert(unsigned(InlineHistoryID) < InlineHistory.size() && - "Invalid inline history ID"); - if (InlineHistory[InlineHistoryID].first == F) - return true; - InlineHistoryID = InlineHistory[InlineHistoryID].second; - } - return false; -} - InlineAdvisor & InlinerPass::getAdvisor(const ModuleAnalysisManagerCGSCCProxy::Result &MAM, FunctionAnalysisManager &FAM, Module &M) { diff --git a/llvm/lib/Transforms/IPO/ModuleInliner.cpp b/llvm/lib/Transforms/IPO/ModuleInliner.cpp index 3e0bb6d1432b..31c26c9fb8c0 100644 --- a/llvm/lib/Transforms/IPO/ModuleInliner.cpp +++ b/llvm/lib/Transforms/IPO/ModuleInliner.cpp @@ -56,21 +56,6 @@ static cl::opt CtxProfPromoteAlwaysInline( "promotion for that target. If multiple targets for an indirect " "call site fit this description, they are all promoted.")); -/// Return true if the specified inline history ID -/// indicates an inline history that includes the specified function. -static bool inlineHistoryIncludes( - Function *F, int InlineHistoryID, - const SmallVectorImpl> &InlineHistory) { - while (InlineHistoryID != -1) { - assert(unsigned(InlineHistoryID) < InlineHistory.size() && - "Invalid inline history ID"); - if (InlineHistory[InlineHistoryID].first == F) - return true; - InlineHistoryID = InlineHistory[InlineHistoryID].second; - } - return false; -} - InlineAdvisor &ModuleInlinerPass::getAdvisor(const ModuleAnalysisManager &MAM, FunctionAnalysisManager &FAM, Module &M) { diff --git a/llvm/lib/Transforms/Utils/CodeExtractor.cpp b/llvm/lib/Transforms/Utils/CodeExtractor.cpp index 74bb7efef40e..8faf64a60997 100644 --- a/llvm/lib/Transforms/Utils/CodeExtractor.cpp +++ b/llvm/lib/Transforms/Utils/CodeExtractor.cpp @@ -938,6 +938,7 @@ Function *CodeExtractor::constructFunctionDeclaration( case Attribute::AlwaysInline: case Attribute::Cold: case Attribute::DisableSanitizerInstrumentation: + case Attribute::Flatten: case Attribute::FnRetThunkExtern: case Attribute::Hot: case Attribute::HybridPatchable: diff --git a/llvm/lib/Transforms/Utils/InlineFunction.cpp b/llvm/lib/Transforms/Utils/InlineFunction.cpp index 8866c9945331..0c0d0f22d036 100644 --- a/llvm/lib/Transforms/Utils/InlineFunction.cpp +++ b/llvm/lib/Transforms/Utils/InlineFunction.cpp @@ -3444,3 +3444,16 @@ llvm::InlineResult llvm::InlineFunction(CallBase &CB, InlineFunctionInfo &IFI, return Result; } + +bool llvm::inlineHistoryIncludes( + Function *F, int InlineHistoryID, + ArrayRef> InlineHistory) { + while (InlineHistoryID != -1) { + assert(unsigned(InlineHistoryID) < InlineHistory.size() && + "Invalid inline history ID"); + if (InlineHistory[InlineHistoryID].first == F) + return true; + InlineHistoryID = InlineHistory[InlineHistoryID].second; + } + return false; +} diff --git a/llvm/test/Bitcode/attributes.ll b/llvm/test/Bitcode/attributes.ll index 4f234d7a4007..21712fae7eec 100644 --- a/llvm/test/Bitcode/attributes.ll +++ b/llvm/test/Bitcode/attributes.ll @@ -526,6 +526,11 @@ define void @f_no_create_undef_or_poison() nocreateundeforpoison { ret void; } +; CHECK: define void @f_flatten() [[FLATTEN:#[0-9]+]] +define void @f_flatten() flatten { + ret void; +} + ; CHECK: define void @f87() [[FNRETTHUNKEXTERN:#[0-9]+]] define void @f87() fn_ret_thunk_extern { ret void } @@ -644,6 +649,7 @@ define void @dead_on_return_sized(ptr dead_on_return(4) %p) { ; CHECK: attributes #54 = { sanitize_realtime_blocking } ; CHECK: attributes #55 = { sanitize_alloc_token } ; CHECK: attributes #56 = { nocreateundeforpoison } +; CHECK: attributes [[FLATTEN]] = { flatten } ; CHECK: attributes [[FNRETTHUNKEXTERN]] = { fn_ret_thunk_extern } ; CHECK: attributes [[SKIPPROFILE]] = { skipprofile } ; CHECK: attributes [[OPTDEBUG]] = { optdebug } diff --git a/llvm/test/Transforms/Inline/AArch64/flatten-sme.ll b/llvm/test/Transforms/Inline/AArch64/flatten-sme.ll new file mode 100644 index 000000000000..e3d302f5eca7 --- /dev/null +++ b/llvm/test/Transforms/Inline/AArch64/flatten-sme.ll @@ -0,0 +1,91 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --version 5 +; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sme -S -passes=always-inline | FileCheck %s --check-prefixes=CHECK,ALWAYS +; RUN: opt < %s -mtriple=aarch64-unknown-linux-gnu -mattr=+sme -S -passes=inline | FileCheck %s --check-prefixes=CHECK,INLINE + +; Test that flatten attribute respects ABI restrictions for SME. +; Streaming callee cannot be inlined into non-streaming caller. +; new_za callee cannot be inlined at all. + +define internal i32 @streaming_callee() "aarch64_pstate_sm_enabled" { +; CHECK-LABEL: define internal i32 @streaming_callee( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: ret i32 42 +; + ret i32 42 +} + +define internal i32 @new_za_callee() "aarch64_new_za" { +; CHECK-LABEL: define internal i32 @new_za_callee( +; CHECK-SAME: ) #[[ATTR1:[0-9]+]] { +; CHECK-NEXT: ret i32 100 +; + ret i32 100 +} + +define internal i32 @normal_callee() { +; ALWAYS-LABEL: define internal i32 @normal_callee( +; ALWAYS-SAME: ) #[[ATTR2:[0-9]+]] { +; ALWAYS-NEXT: ret i32 50 +; + ret i32 50 +} + +; Streaming callee -> non-streaming caller: should NOT be inlined (ABI violation). +define i32 @test_streaming_not_inlined() flatten { +; ALWAYS-LABEL: define i32 @test_streaming_not_inlined( +; ALWAYS-SAME: ) #[[ATTR3:[0-9]+]] { +; ALWAYS-NEXT: [[R:%.*]] = call i32 @streaming_callee() +; ALWAYS-NEXT: ret i32 [[R]] +; +; INLINE-LABEL: define i32 @test_streaming_not_inlined( +; INLINE-SAME: ) #[[ATTR2:[0-9]+]] { +; INLINE-NEXT: [[R:%.*]] = call i32 @streaming_callee() +; INLINE-NEXT: ret i32 [[R]] +; + %r = call i32 @streaming_callee() + ret i32 %r +} + +; new_za callee: should NOT be inlined (ABI violation - callee allocates new ZA). +define i32 @test_new_za_not_inlined() flatten { +; ALWAYS-LABEL: define i32 @test_new_za_not_inlined( +; ALWAYS-SAME: ) #[[ATTR3]] { +; ALWAYS-NEXT: [[R:%.*]] = call i32 @new_za_callee() +; ALWAYS-NEXT: ret i32 [[R]] +; +; INLINE-LABEL: define i32 @test_new_za_not_inlined( +; INLINE-SAME: ) #[[ATTR2]] { +; INLINE-NEXT: [[R:%.*]] = call i32 @new_za_callee() +; INLINE-NEXT: ret i32 [[R]] +; + %r = call i32 @new_za_callee() + ret i32 %r +} + +; Streaming caller -> streaming callee: should be inlined (compatible). +define i32 @test_streaming_to_streaming() flatten "aarch64_pstate_sm_enabled" { +; ALWAYS-LABEL: define i32 @test_streaming_to_streaming( +; ALWAYS-SAME: ) #[[ATTR4:[0-9]+]] { +; ALWAYS-NEXT: ret i32 42 +; +; INLINE-LABEL: define i32 @test_streaming_to_streaming( +; INLINE-SAME: ) #[[ATTR3:[0-9]+]] { +; INLINE-NEXT: ret i32 42 +; + %r = call i32 @streaming_callee() + ret i32 %r +} + +; Non-streaming caller -> non-streaming callee: should be inlined. +define i32 @test_normal_inlined() flatten { +; ALWAYS-LABEL: define i32 @test_normal_inlined( +; ALWAYS-SAME: ) #[[ATTR3]] { +; ALWAYS-NEXT: ret i32 50 +; +; INLINE-LABEL: define i32 @test_normal_inlined( +; INLINE-SAME: ) #[[ATTR2]] { +; INLINE-NEXT: ret i32 50 +; + %r = call i32 @normal_callee() + ret i32 %r +} diff --git a/llvm/test/Transforms/Inline/flatten.ll b/llvm/test/Transforms/Inline/flatten.ll new file mode 100644 index 000000000000..355739a99dac --- /dev/null +++ b/llvm/test/Transforms/Inline/flatten.ll @@ -0,0 +1,207 @@ +; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --scrub-attributes --version 6 +; RUN: opt -passes=always-inline -S < %s | FileCheck %s --check-prefixes=CHECK,ALWAYS +; RUN: opt -passes=inline -S < %s | FileCheck %s --check-prefixes=CHECK,INLINE +; RUN: opt -passes='cgscc(inline)' -S < %s | FileCheck %s --check-prefixes=CHECK,MANDATORY +; RUN: opt -passes=always-inline -pass-remarks-missed=inline -S %s -o /dev/null 2>&1 | FileCheck --check-prefix=REMARK %s + +; Test that the flatten attribute recursively inlines all calls. + +; Multiple levels are inlined. +define internal i32 @leaf() { +; ALWAYS-LABEL: define internal i32 @leaf() { +; ALWAYS-NEXT: ret i32 42 +; + ret i32 42 +} + +define internal i32 @middle() { +; ALWAYS-LABEL: define internal i32 @middle() { +; ALWAYS-NEXT: [[R:%.*]] = call i32 @leaf() +; ALWAYS-NEXT: ret i32 [[R]] +; + %r = call i32 @leaf() + ret i32 %r +} + +define i32 @test_multilevel() flatten { +; CHECK-LABEL: define i32 @test_multilevel( +; CHECK-SAME: ) #[[ATTR0:[0-9]+]] { +; CHECK-NEXT: ret i32 42 +; + %r = call i32 @middle() + ret i32 %r +} + +; Functions with invoke are inlined. +declare i32 @__gxx_personality_v0(...) +declare void @may_throw() + +define internal i32 @callee_with_invoke() personality ptr @__gxx_personality_v0 { +; ALWAYS-LABEL: define internal i32 @callee_with_invoke() personality ptr @__gxx_personality_v0 { +; ALWAYS-NEXT: [[ENTRY:.*:]] +; ALWAYS-NEXT: invoke void @may_throw() +; ALWAYS-NEXT: to label %[[CONT:.*]] unwind label %[[LPAD:.*]] +; ALWAYS: [[CONT]]: +; ALWAYS-NEXT: ret i32 100 +; ALWAYS: [[LPAD]]: +; ALWAYS-NEXT: [[LP:%.*]] = landingpad { ptr, i32 } +; ALWAYS-NEXT: cleanup +; ALWAYS-NEXT: resume { ptr, i32 } [[LP]] +; +entry: + invoke void @may_throw() to label %cont unwind label %lpad +cont: + ret i32 100 +lpad: + %lp = landingpad { ptr, i32 } cleanup + resume { ptr, i32 } %lp +} + +define i32 @test_invoke() flatten personality ptr @__gxx_personality_v0 { +; CHECK-LABEL: define i32 @test_invoke( +; CHECK-SAME: ) #[[ATTR0]] personality ptr @__gxx_personality_v0 { +; CHECK-NEXT: [[ENTRY:.*:]] +; CHECK-NEXT: invoke void @may_throw() +; CHECK-NEXT: to label %[[CALLEE_WITH_INVOKE_EXIT:.*]] unwind label %[[LPAD_I:.*]] +; CHECK: [[LPAD_I]]: +; CHECK-NEXT: [[LP_I:%.*]] = landingpad { ptr, i32 } +; CHECK-NEXT: cleanup +; CHECK-NEXT: resume { ptr, i32 } [[LP_I]] +; CHECK: [[CALLEE_WITH_INVOKE_EXIT]]: +; CHECK-NEXT: ret i32 100 +; +entry: + %r = call i32 @callee_with_invoke() + ret i32 %r +} + +; Declaration without definition is not inlined. +declare i32 @external_func() + +define i32 @test_declaration() flatten { +; CHECK-LABEL: define i32 @test_declaration( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[R:%.*]] = call i32 @external_func() +; CHECK-NEXT: ret i32 [[R]] +; + %r = call i32 @external_func() + ret i32 %r +} + +; Inlined callee that calls a declaration - the declaration should remain after flattening. +define internal i32 @calls_external() { +; ALWAYS-LABEL: define internal i32 @calls_external() { +; ALWAYS-NEXT: [[R:%.*]] = call i32 @external_func() +; ALWAYS-NEXT: ret i32 [[R]] +; + %r = call i32 @external_func() + ret i32 %r +} + +define i32 @test_inline_then_declaration() flatten { +; CHECK-LABEL: define i32 @test_inline_then_declaration( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[R_I:%.*]] = call i32 @external_func() +; CHECK-NEXT: ret i32 [[R_I]] +; + %r = call i32 @calls_external() + ret i32 %r +} + +; Indirect calls are not inlined. +define internal i32 @target_func() { +; CHECK-LABEL: define internal i32 @target_func() { +; CHECK-NEXT: ret i32 99 +; + ret i32 99 +} + +define i32 @test_indirect(ptr %func_ptr) flatten { +; CHECK-LABEL: define i32 @test_indirect( +; CHECK-SAME: ptr [[FUNC_PTR:%.*]]) #[[ATTR0]] { +; CHECK-NEXT: [[R:%.*]] = call i32 [[FUNC_PTR]]() +; CHECK-NEXT: ret i32 [[R]] +; + %r = call i32 %func_ptr() + ret i32 %r +} + +; Direct recursion back to flattened function. +; The callee calls the flattened function - should not cause infinite inlining. +define internal i32 @calls_flattened_func() { +; ALWAYS-LABEL: define internal i32 @calls_flattened_func() { +; ALWAYS-NEXT: [[R:%.*]] = call i32 @test_direct_recursion() +; ALWAYS-NEXT: ret i32 [[R]] +; + %r = call i32 @test_direct_recursion() + ret i32 %r +} + +define i32 @test_direct_recursion() flatten { +; The call to calls_flattened_func should be inlined, but the recursive call back +; to test_direct_recursion should remain. +; CHECK-LABEL: define i32 @test_direct_recursion( +; CHECK-SAME: ) #[[ATTR0]] { +; CHECK-NEXT: [[R_I:%.*]] = call i32 @test_direct_recursion() +; CHECK-NEXT: ret i32 [[R_I]] +; + %r = call i32 @calls_flattened_func() + ret i32 %r +} + +; Mutual recursion (A calls B, B calls A). +; Should inline once but not infinitely. +define internal i32 @mutual_a() { +; ALWAYS-LABEL: define internal i32 @mutual_a() { +; ALWAYS-NEXT: [[R:%.*]] = call i32 @mutual_b() +; ALWAYS-NEXT: ret i32 [[R]] +; +; INLINE-LABEL: define internal i32 @mutual_a() { +; INLINE-NEXT: [[R_I:%.*]] = call i32 @mutual_a() +; INLINE-NEXT: ret i32 [[R_I]] +; +; MANDATORY-LABEL: define internal i32 @mutual_a() { +; MANDATORY-NEXT: [[R:%.*]] = call i32 @mutual_b() +; MANDATORY-NEXT: ret i32 [[R]] +; + %r = call i32 @mutual_b() + ret i32 %r +} + +define internal i32 @mutual_b() { +; ALWAYS-LABEL: define internal i32 @mutual_b() { +; ALWAYS-NEXT: [[R:%.*]] = call i32 @mutual_a() +; ALWAYS-NEXT: ret i32 [[R]] +; +; MANDATORY-LABEL: define internal i32 @mutual_b() { +; MANDATORY-NEXT: [[R:%.*]] = call i32 @mutual_a() +; MANDATORY-NEXT: ret i32 [[R]] +; + %r = call i32 @mutual_a() + ret i32 %r +} + +define i32 @test_mutual_recursion() flatten { +; After inlining mutual_a, we get call to mutual_b. +; After inlining mutual_b, we get call to mutual_a which should remain (skipped due to inline history). +; ALWAYS-LABEL: define i32 @test_mutual_recursion( +; ALWAYS-SAME: ) #[[ATTR0]] { +; ALWAYS-NEXT: [[R_I1:%.*]] = call i32 @mutual_a() +; ALWAYS-NEXT: ret i32 [[R_I1]] +; +; INLINE-LABEL: define i32 @test_mutual_recursion( +; INLINE-SAME: ) #[[ATTR0]] { +; INLINE-NEXT: [[R:%.*]] = call i32 @mutual_a() +; INLINE-NEXT: ret i32 [[R]] +; +; MANDATORY-LABEL: define i32 @test_mutual_recursion( +; MANDATORY-SAME: ) #[[ATTR0]] { +; MANDATORY-NEXT: [[R_I1:%.*]] = call i32 @mutual_a() +; MANDATORY-NEXT: ret i32 [[R_I1]] +; + %r = call i32 @mutual_a() + ret i32 %r +} + +; Check that optimization remark is emitted for recursive calls during flattening. +; REMARK: remark: {{.*}} 'test_direct_recursion' is not inlined into 'test_direct_recursion': recursive call during flattening