shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity

Don't disable loop unroll for vectorized loops on AMDGPU target

Browse Source 
We've got a performance regression after the https://reviews.llvm.org/D115261.
Despite the loop being vectorized unroll is still required.

Reviewed By: rampitec

Differential Revision: https://reviews.llvm.org/D149281
This commit is contained in:
Alexander Timofeev 2023-04-26 20:58:17 +02:00
parent dd16cd731d
commit bad4de1ae7
4 changed files with 43 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
llvm/include/llvm/Analysis/TargetTransformInfo.h
View File

@ -569,6 +569,8 @@ public:
/// Don't allow loop unrolling to simulate more than this number of
/// iterations when checking full unroll profitability
unsigned MaxIterationsCountToAnalyze;
/// Don't disable runtime unroll for the loops which were vectorized.
bool UnrollVectorizedLoop = false;
};
/// Get target-customized preferences for the generic loop unrolling

3
llvm/lib/Target/AMDGPU/AMDGPUTargetTransformInfo.cpp
View File

@ -113,6 +113,9 @@ void AMDGPUTTIImpl::getUnrollingPreferences(Loop *L, ScalarEvolution &SE,
// manipulations in average.
UP.BEInsns += 3;
// We want to run unroll even for the loops which have been vectorized.
UP.UnrollVectorizedLoop = true;
// TODO: Do we want runtime unrolling?
// Maximum alloca size than can fit registers. Reserve 16 registers.

5
llvm/lib/Transforms/Vectorize/LoopVectorize.cpp
View File

@ -7743,7 +7743,10 @@ SCEV2ValueTy LoopVectorizationPlanner::executePlan(
LoopVectorizeHints Hints(L, true, *ORE);
Hints.setAlreadyVectorized();
}
AddRuntimeUnrollDisableMetaData(L);
TargetTransformInfo::UnrollingPreferences UP;
TTI.getUnrollingPreferences(L, *PSE.getSE(), UP, ORE);
if (!UP.UnrollVectorizedLoop || CanonicalIVStartValue)
AddRuntimeUnrollDisableMetaData(L);
// 3. Fix the vectorized code: take care of header phi's, live-outs,
// predication, updating analyses.

34
llvm/test/CodeGen/AMDGPU/vectorize-unroll-metadata.ll Normal file
View File

@ -0,0 +1,34 @@
; RUN: opt -mtriple=amdgcn-- -mcpu=gfx90a -passes=loop-vectorize %s -S -o - | FileCheck %s
; CHECK-LABEL: @test
; CHECK-LABEL: vector.body:
; CHECK: br i1 %{{[0-9]+}}, label %middle.block, label %vector.body, !llvm.loop !0
; CHECK-LABEL: middle.block:
; CHECK-LABEL: scalar.ph:
; CHECK-LABEL: loop.body:
; CHECK: br i1 %cond, label %exit, label %loop.body, !llvm.loop !2
; CHECK: !0 = distinct !{!0, !1}
; CHECK: !1 = !{!"llvm.loop.isvectorized", i32 1}
; CHECK: !2 = distinct !{!2, !3, !1}
; CHECK: !3 = !{!"llvm.loop.unroll.runtime.disable"}
define amdgpu_kernel void @test(ptr addrspace(1) %out, ptr addrspace(3) %lds, i32 %n) {
entry:
br label %loop.body
loop.body:
%counter = phi i32 [0, %entry], [%inc, %loop.body]
%ptr_lds = getelementptr i32, ptr addrspace(3) %lds, i32 %counter
%val = load i32, ptr addrspace(3) %ptr_lds
%ptr_out = getelementptr i32, ptr addrspace(1) %out, i32 %counter
store i32 %val, ptr addrspace(1) %ptr_out
%inc = add i32 %counter, 1
%cond = icmp sge i32 %counter, %n
br i1 %cond, label %exit, label %loop.body
exit:
ret void
}