diff --git a/llvm/include/llvm/Analysis/TargetLibraryInfo.h b/llvm/include/llvm/Analysis/TargetLibraryInfo.h index a2986c737466..629b126db17c 100644 --- a/llvm/include/llvm/Analysis/TargetLibraryInfo.h +++ b/llvm/include/llvm/Analysis/TargetLibraryInfo.h @@ -416,9 +416,9 @@ public: case LibFunc_sqrtl_finite: case LibFunc_strcpy: case LibFunc_stpcpy: case LibFunc_strlen: case LibFunc_strnlen: case LibFunc_strstr: case LibFunc_memchr: - case LibFunc_mempcpy: case LibFunc_tan: case LibFunc_tanf: - case LibFunc_tanl: case LibFunc_tanh: case LibFunc_tanhf: - case LibFunc_tanhl: + case LibFunc_memccpy: case LibFunc_mempcpy: case LibFunc_tan: + case LibFunc_tanf: case LibFunc_tanl: case LibFunc_tanh: + case LibFunc_tanhf: case LibFunc_tanhl: // clang-format on return true; } diff --git a/llvm/include/llvm/CodeGen/SelectionDAG.h b/llvm/include/llvm/CodeGen/SelectionDAG.h index 4a2f98d36a3b..b2a9e076fb90 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAG.h +++ b/llvm/include/llvm/CodeGen/SelectionDAG.h @@ -1292,6 +1292,12 @@ public: /// stack arguments from being clobbered. LLVM_ABI SDValue getStackArgumentTokenFactor(SDValue Chain); + /// Lower a memccpy operation into a target library call and return the + /// resulting chain and call result as SelectionDAG SDValues. + LLVM_ABI std::pair + getMemccpy(SDValue Chain, const SDLoc &dl, SDValue Dst, SDValue Src, + SDValue C, SDValue Size, const CallInst *CI); + /// Lower a memcmp operation into a target library call and return the /// resulting chain and call result as SelectionDAG SDValues. LLVM_ABI std::pair getMemcmp(SDValue Chain, const SDLoc &dl, diff --git a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h index 508514a00781..64f63d5b2190 100644 --- a/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h +++ b/llvm/include/llvm/CodeGen/SelectionDAGTargetInfo.h @@ -122,6 +122,17 @@ public: return std::make_pair(SDValue(), SDValue()); } + /// Emit target-specific code that performs a memccpy, in cases where that is + /// faster than a libcall. The first returned SDValue is the result of the + /// memccpy and the second is the chain. Both SDValues can be null if a normal + /// libcall should be used. + virtual std::pair + EmitTargetCodeForMemccpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, + SDValue Dst, SDValue Src, SDValue C, SDValue Size, + const CallInst *CI) const { + return std::make_pair(SDValue(), SDValue()); + } + /// Emit target-specific code that performs a memcmp/bcmp, in cases where that is /// faster than a libcall. The first returned SDValue is the result of the /// memcmp and the second is the chain. Both SDValues can be null if a normal diff --git a/llvm/include/llvm/IR/RuntimeLibcalls.td b/llvm/include/llvm/IR/RuntimeLibcalls.td index 5773705c58d6..e4a926d3cb1d 100644 --- a/llvm/include/llvm/IR/RuntimeLibcalls.td +++ b/llvm/include/llvm/IR/RuntimeLibcalls.td @@ -3161,6 +3161,7 @@ defset list PPCRuntimeLibcalls = { } defset list PPC64AIXCallList = { + def ___memccpy64 : RuntimeLibcallImpl; def ___memcmp64 : RuntimeLibcallImpl; def ___memmove64 : RuntimeLibcallImpl; def ___memset64 : RuntimeLibcallImpl; @@ -3172,6 +3173,7 @@ defset list PPC64AIXCallList = { } defset list PPC32AIXCallList = { + def ___memccpy : RuntimeLibcallImpl; def ___memcmp : RuntimeLibcallImpl; def ___memmove : RuntimeLibcallImpl; def ___memset : RuntimeLibcallImpl; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp index b7f78439505f..4a2bd811b521 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAG.cpp @@ -9387,6 +9387,22 @@ std::pair SelectionDAG::getStrstr(SDValue Chain, RTLIB::STRSTR, this, TLI); } +std::pair SelectionDAG::getMemccpy(SDValue Chain, + const SDLoc &dl, + SDValue Dst, SDValue Src, + SDValue C, SDValue Size, + const CallInst *CI) { + PointerType *PT = PointerType::getUnqual(*getContext()); + + TargetLowering::ArgListTy Args = { + {Dst, PT}, + {Src, PT}, + {C, Type::getInt32Ty(*getContext())}, + {Size, getDataLayout().getIntPtrType(*getContext())}}; + return getRuntimeCallSDValueHelper(Chain, dl, std::move(Args), CI, + RTLIB::MEMCCPY, this, TLI); +} + std::pair SelectionDAG::getMemcmp(SDValue Chain, const SDLoc &dl, SDValue Mem0, SDValue Mem1, SDValue Size, const CallInst *CI) { diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp index e21bfa8498c8..392e53b99c64 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.cpp @@ -9399,6 +9399,26 @@ bool SelectionDAGBuilder::visitMemChrCall(const CallInst &I) { return false; } +/// See if we can lower a memccpy call into an optimized form. If so, return +/// true and lower it, otherwise return false and it will be lowered like a +/// normal call. +/// The caller already checked that \p I calls the appropriate LibFunc with a +/// correct prototype. +bool SelectionDAGBuilder::visitMemCCpyCall(const CallInst &I) { + const SelectionDAGTargetInfo &TSI = DAG.getSelectionDAGInfo(); + std::pair Res = TSI.EmitTargetCodeForMemccpy( + DAG, getCurSDLoc(), DAG.getRoot(), getValue(I.getArgOperand(0)), + getValue(I.getArgOperand(1)), getValue(I.getArgOperand(2)), + getValue(I.getArgOperand(3)), &I); + + if (Res.first) { + processIntegerCallValue(I, Res.first, true); + PendingLoads.push_back(Res.second); + return true; + } + return false; +} + /// See if we can lower a mempcpy call into an optimized form. If so, return /// true and lower it. Otherwise return false, and it will be lowered like a /// normal call. @@ -9730,6 +9750,10 @@ void SelectionDAGBuilder::visitCall(const CallInst &I) { if (visitMemCmpBCmpCall(I)) return; break; + case LibFunc_memccpy: + if (visitMemCCpyCall(I)) + return; + break; case LibFunc_mempcpy: if (visitMemPCpyCall(I)) return; diff --git a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h index f8aecea25b3d..a81a240b057d 100644 --- a/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h +++ b/llvm/lib/CodeGen/SelectionDAG/SelectionDAGBuilder.h @@ -627,6 +627,7 @@ private: void visitPHI(const PHINode &I); void visitCall(const CallInst &I); bool visitMemCmpBCmpCall(const CallInst &I); + bool visitMemCCpyCall(const CallInst &I); bool visitMemPCpyCall(const CallInst &I); bool visitMemChrCall(const CallInst &I); bool visitStrCpyCall(const CallInst &I, bool isStpcpy); diff --git a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp index dc8434fc703b..d0c685d95d2b 100644 --- a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp +++ b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.cpp @@ -75,6 +75,13 @@ void PPCSelectionDAGInfo::verifyTargetNode(const SelectionDAG &DAG, SelectionDAGGenTargetInfo::verifyTargetNode(DAG, N); } +std::pair PPCSelectionDAGInfo::EmitTargetCodeForMemccpy( + SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Dst, SDValue Src, + SDValue C, SDValue Size, const CallInst *CI) const { + return DAG.getMemccpy(Chain, dl, Dst, Src, C, Size, CI); + ; +} + std::pair PPCSelectionDAGInfo::EmitTargetCodeForMemcmp( SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, SDValue Op1, SDValue Op2, SDValue Op3, const CallInst *CI) const { diff --git a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h index ae6ae85d0b0e..17d544a8d558 100644 --- a/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h +++ b/llvm/lib/Target/PowerPC/PPCSelectionDAGInfo.h @@ -71,6 +71,10 @@ public: void verifyTargetNode(const SelectionDAG &DAG, const SDNode *N) const override; + std::pair + EmitTargetCodeForMemccpy(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, + SDValue Dst, SDValue Src, SDValue C, SDValue Size, + const CallInst *CI) const override; std::pair EmitTargetCodeForMemcmp(SelectionDAG &DAG, const SDLoc &dl, SDValue Chain, diff --git a/llvm/test/CodeGen/PowerPC/milicode32.ll b/llvm/test/CodeGen/PowerPC/milicode32.ll index 491f32ee6505..2c7e5c986713 100644 --- a/llvm/test/CodeGen/PowerPC/milicode32.ll +++ b/llvm/test/CodeGen/PowerPC/milicode32.ll @@ -312,7 +312,7 @@ define ptr @test_memccpy(ptr noalias noundef %dst, ptr noalias noundef %src, i32 ; CHECK-AIX-32-P9-NEXT: mflr r0 ; CHECK-AIX-32-P9-NEXT: stwu r1, -64(r1) ; CHECK-AIX-32-P9-NEXT: stw r0, 72(r1) -; CHECK-AIX-32-P9-NEXT: bl .memccpy[PR] +; CHECK-AIX-32-P9-NEXT: bl .___memccpy[PR] ; CHECK-AIX-32-P9-NEXT: nop ; CHECK-AIX-32-P9-NEXT: addi r1, r1, 64 ; CHECK-AIX-32-P9-NEXT: lwz r0, 8(r1) diff --git a/llvm/test/CodeGen/PowerPC/milicode64.ll b/llvm/test/CodeGen/PowerPC/milicode64.ll index 151fe2c52538..8f4c6b495117 100644 --- a/llvm/test/CodeGen/PowerPC/milicode64.ll +++ b/llvm/test/CodeGen/PowerPC/milicode64.ll @@ -425,8 +425,9 @@ define ptr @test_memccpy(ptr noalias noundef %dst, ptr noalias noundef %src, i32 ; CHECK-AIX-64-P9: # %bb.0: # %entry ; CHECK-AIX-64-P9-NEXT: mflr r0 ; CHECK-AIX-64-P9-NEXT: stdu r1, -112(r1) +; CHECK-AIX-64-P9-NEXT: clrldi r5, r5, 32 ; CHECK-AIX-64-P9-NEXT: std r0, 128(r1) -; CHECK-AIX-64-P9-NEXT: bl .memccpy[PR] +; CHECK-AIX-64-P9-NEXT: bl .___memccpy64[PR] ; CHECK-AIX-64-P9-NEXT: nop ; CHECK-AIX-64-P9-NEXT: addi r1, r1, 112 ; CHECK-AIX-64-P9-NEXT: ld r0, 16(r1)