abdcde9bbc
Currently, SLP vectorizer do not care about loops and their trip count. It may lead to inefficient vectorization in some cases. Patch adds loop nest-aware tree building and cost estimation. When it comes to tree building, it now checks that tree do not span across different loop nests. The nodes from other loop nests are immediate buildvector nodes. The cost model adds the knowledge about loop trip count. If it is unknown, the default value is used, controlled by the -slp-cost-loop-min-trip-count=<value> option. The cost of the vector nodes in the loop is multiplied by the number of iteration (trip count), because each vector node will be executed the trip count number of times. This allows better cost estimation. Original Reviewers: jdenny-ornl, vporpo, hiraditya, RKSimon Original PR: https://github.com/llvm/llvm-project/pull/150450 Recommit after revert in c7bd3062f1dac975cf9b706f457b3c55b4bf57ff and in 4e500bd0015042b0cd4b7c87b81caeea06072d24 Reviewers: Pull Request: https://github.com/llvm/llvm-project/pull/187391
212 lines
10 KiB
LLVM
212 lines
10 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
|
|
; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-13 -pass-remarks-output=%t < %s | FileCheck %s
|
|
; RUN: cat %t | FileCheck -check-prefix=YAML %s
|
|
; RUN: opt -S -mtriple=aarch64--linux-gnu -passes=slp-vectorizer -slp-threshold=-13 -pass-remarks-output=%t < %s | FileCheck %s
|
|
; RUN: cat %t | FileCheck -check-prefix=YAML %s
|
|
|
|
; These tests check that we remove from consideration pairs of seed
|
|
; getelementptrs when they are known to have a constant difference. Such pairs
|
|
; are likely not good candidates for vectorization since one can be computed
|
|
; from the other. We use an unprofitable threshold to force vectorization.
|
|
;
|
|
; int getelementptr(int *g, int n, int w, int x, int y, int z) {
|
|
; int sum = 0;
|
|
; for (int i = 0; i < n ; ++i) {
|
|
; sum += g[2*i + w]; sum += g[2*i + x];
|
|
; sum += g[2*i + y]; sum += g[2*i + z];
|
|
; }
|
|
; return sum;
|
|
; }
|
|
;
|
|
|
|
; YAML-LABEL: Function: getelementptr_4x32
|
|
; YAML: --- !Passed
|
|
; YAML-NEXT: Pass: slp-vectorizer
|
|
; YAML-NEXT: Name: VectorizedHorizontalReduction
|
|
; YAML-NEXT: Function: getelementptr_4x32
|
|
; YAML-NEXT: Args:
|
|
; YAML-NEXT: - String: 'Vectorized horizontal reduction with cost '
|
|
; YAML-NEXT: - Cost: '6'
|
|
; YAML-NEXT: - String: ' and with tree size '
|
|
; YAML-NEXT: - TreeSize: '1'
|
|
|
|
; YAML: --- !Passed
|
|
; YAML-NEXT: Pass: slp-vectorizer
|
|
; YAML-NEXT: Name: VectorizedList
|
|
; YAML-NEXT: Function: getelementptr_4x32
|
|
; YAML-NEXT: Args:
|
|
; YAML-NEXT: - String: 'SLP vectorized with cost '
|
|
; YAML-NEXT: - Cost: '12'
|
|
; YAML-NEXT: - String: ' and with tree size '
|
|
; YAML-NEXT: - TreeSize: '3'
|
|
|
|
define i32 @getelementptr_4x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y, i32 %z) {
|
|
; CHECK-LABEL: @getelementptr_4x32(
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
|
|
; CHECK-NEXT: br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
|
|
; CHECK: for.body.preheader:
|
|
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[X:%.*]], i32 1
|
|
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
|
|
; CHECK: for.cond.cleanup.loopexit:
|
|
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
|
|
; CHECK: for.cond.cleanup:
|
|
; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[ADD16:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
|
|
; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
|
|
; CHECK: for.body:
|
|
; CHECK-NEXT: [[TMP15:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[SUM_032:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[ADD16]], [[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[T4:%.*]] = shl nsw i32 [[TMP15]], 1
|
|
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i32 0
|
|
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP0]]
|
|
; CHECK-NEXT: [[TMP12:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
|
|
; CHECK-NEXT: [[ARRAYIDX:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i32 [[TMP12]]
|
|
; CHECK-NEXT: [[T6:%.*]] = load i32, ptr [[ARRAYIDX]], align 4
|
|
; CHECK-NEXT: [[TMP11:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
|
|
; CHECK-NEXT: [[ARRAYIDX5:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP11]]
|
|
; CHECK-NEXT: [[T8:%.*]] = load i32, ptr [[ARRAYIDX5]], align 4
|
|
; CHECK-NEXT: [[TMP13:%.*]] = add nsw i32 [[T4]], [[Y:%.*]]
|
|
; CHECK-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP13]]
|
|
; CHECK-NEXT: [[T10:%.*]] = load i32, ptr [[ARRAYIDX10]], align 4
|
|
; CHECK-NEXT: [[TMP14:%.*]] = add nsw i32 [[T4]], [[Z:%.*]]
|
|
; CHECK-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP14]]
|
|
; CHECK-NEXT: [[T12:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
|
|
; CHECK-NEXT: [[TMP6:%.*]] = insertelement <4 x i32> poison, i32 [[T6]], i32 0
|
|
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> [[TMP6]], i32 [[T8]], i32 1
|
|
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[T10]], i32 2
|
|
; CHECK-NEXT: [[TMP9:%.*]] = insertelement <4 x i32> [[TMP8]], i32 [[T12]], i32 3
|
|
; CHECK-NEXT: [[TMP10:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP9]])
|
|
; CHECK-NEXT: [[ADD16]] = add i32 [[TMP10]], [[SUM_032]]
|
|
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[TMP15]], 1
|
|
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
|
|
;
|
|
entry:
|
|
%cmp31 = icmp sgt i32 %n, 0
|
|
br i1 %cmp31, label %for.body.preheader, label %for.cond.cleanup
|
|
|
|
for.body.preheader:
|
|
br label %for.body
|
|
|
|
for.cond.cleanup.loopexit:
|
|
br label %for.cond.cleanup
|
|
|
|
for.cond.cleanup:
|
|
%sum.0.lcssa = phi i32 [ 0, %entry ], [ %add16, %for.cond.cleanup.loopexit ]
|
|
ret i32 %sum.0.lcssa
|
|
|
|
for.body:
|
|
%indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
|
|
%sum.032 = phi i32 [ 0, %for.body.preheader ], [ %add16, %for.body ]
|
|
%t4 = shl nsw i32 %indvars.iv, 1
|
|
%t5 = add nsw i32 %t4, 0
|
|
%arrayidx = getelementptr inbounds i32, ptr %g, i32 %t5
|
|
%t6 = load i32, ptr %arrayidx, align 4
|
|
%add1 = add nsw i32 %t6, %sum.032
|
|
%t7 = add nsw i32 %t4, %x
|
|
%arrayidx5 = getelementptr inbounds i32, ptr %g, i32 %t7
|
|
%t8 = load i32, ptr %arrayidx5, align 4
|
|
%add6 = add nsw i32 %add1, %t8
|
|
%t9 = add nsw i32 %t4, %y
|
|
%arrayidx10 = getelementptr inbounds i32, ptr %g, i32 %t9
|
|
%t10 = load i32, ptr %arrayidx10, align 4
|
|
%add11 = add nsw i32 %add6, %t10
|
|
%t11 = add nsw i32 %t4, %z
|
|
%arrayidx15 = getelementptr inbounds i32, ptr %g, i32 %t11
|
|
%t12 = load i32, ptr %arrayidx15, align 4
|
|
%add16 = add nsw i32 %add11, %t12
|
|
%indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
|
|
%exitcond = icmp eq i32 %indvars.iv.next , %n
|
|
br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
|
|
}
|
|
|
|
; YAML-LABEL: Function: getelementptr_2x32
|
|
; YAML: --- !Passed
|
|
; YAML: Pass: slp-vectorizer
|
|
; YAML: Name: VectorizedList
|
|
; YAML: Function: getelementptr_2x32
|
|
; YAML: Args:
|
|
; YAML: - String: 'SLP vectorized with cost '
|
|
; YAML: - Cost: '12'
|
|
; YAML-NEXT: - String: ' and with tree size '
|
|
; YAML-NEXT: - TreeSize: '3'
|
|
|
|
define i32 @getelementptr_2x32(ptr nocapture readonly %g, i32 %n, i32 %x, i32 %y, i32 %z) {
|
|
; CHECK-LABEL: @getelementptr_2x32(
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[CMP31:%.*]] = icmp sgt i32 [[N:%.*]], 0
|
|
; CHECK-NEXT: br i1 [[CMP31]], label [[FOR_BODY_PREHEADER:%.*]], label [[FOR_COND_CLEANUP:%.*]]
|
|
; CHECK: for.body.preheader:
|
|
; CHECK-NEXT: [[TMP0:%.*]] = insertelement <2 x i32> <i32 0, i32 poison>, i32 [[Y:%.*]], i32 1
|
|
; CHECK-NEXT: br label [[FOR_BODY:%.*]]
|
|
; CHECK: for.cond.cleanup.loopexit:
|
|
; CHECK-NEXT: br label [[FOR_COND_CLEANUP]]
|
|
; CHECK: for.cond.cleanup:
|
|
; CHECK-NEXT: [[SUM_0_LCSSA:%.*]] = phi i32 [ 0, [[ENTRY:%.*]] ], [ [[OP_RDX:%.*]], [[FOR_COND_CLEANUP_LOOPEXIT:%.*]] ]
|
|
; CHECK-NEXT: ret i32 [[SUM_0_LCSSA]]
|
|
; CHECK: for.body:
|
|
; CHECK-NEXT: [[TMP12:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[INDVARS_IV_NEXT:%.*]], [[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[SUM_032:%.*]] = phi i32 [ 0, [[FOR_BODY_PREHEADER]] ], [ [[OP_RDX]], [[FOR_BODY]] ]
|
|
; CHECK-NEXT: [[T4:%.*]] = shl nsw i32 [[TMP12]], 1
|
|
; CHECK-NEXT: [[TMP1:%.*]] = insertelement <2 x i32> poison, i32 [[T4]], i32 0
|
|
; CHECK-NEXT: [[TMP2:%.*]] = shufflevector <2 x i32> [[TMP1]], <2 x i32> poison, <2 x i32> zeroinitializer
|
|
; CHECK-NEXT: [[TMP3:%.*]] = add nsw <2 x i32> [[TMP2]], [[TMP0]]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = extractelement <2 x i32> [[TMP3]], i32 0
|
|
; CHECK-NEXT: [[ARRAYIDX11:%.*]] = getelementptr inbounds i32, ptr [[G:%.*]], i32 [[TMP9]]
|
|
; CHECK-NEXT: [[TMP5:%.*]] = load <2 x i32>, ptr [[ARRAYIDX11]], align 4
|
|
; CHECK-NEXT: [[TMP6:%.*]] = extractelement <2 x i32> [[TMP3]], i32 1
|
|
; CHECK-NEXT: [[ARRAYIDX10:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP6]]
|
|
; CHECK-NEXT: [[T10:%.*]] = load i32, ptr [[ARRAYIDX10]], align 4
|
|
; CHECK-NEXT: [[TMP10:%.*]] = add nsw i32 [[T4]], [[Z:%.*]]
|
|
; CHECK-NEXT: [[ARRAYIDX15:%.*]] = getelementptr inbounds i32, ptr [[G]], i32 [[TMP10]]
|
|
; CHECK-NEXT: [[T12:%.*]] = load i32, ptr [[ARRAYIDX15]], align 4
|
|
; CHECK-NEXT: [[TMP7:%.*]] = insertelement <4 x i32> poison, i32 [[T10]], i32 2
|
|
; CHECK-NEXT: [[TMP8:%.*]] = insertelement <4 x i32> [[TMP7]], i32 [[T12]], i32 3
|
|
; CHECK-NEXT: [[TMP13:%.*]] = shufflevector <2 x i32> [[TMP5]], <2 x i32> poison, <4 x i32> <i32 0, i32 1, i32 poison, i32 poison>
|
|
; CHECK-NEXT: [[TMP14:%.*]] = shufflevector <4 x i32> [[TMP8]], <4 x i32> [[TMP13]], <4 x i32> <i32 4, i32 5, i32 2, i32 3>
|
|
; CHECK-NEXT: [[TMP11:%.*]] = call i32 @llvm.vector.reduce.add.v4i32(<4 x i32> [[TMP14]])
|
|
; CHECK-NEXT: [[OP_RDX]] = add i32 [[TMP11]], [[SUM_032]]
|
|
; CHECK-NEXT: [[INDVARS_IV_NEXT]] = add nuw nsw i32 [[TMP12]], 1
|
|
; CHECK-NEXT: [[EXITCOND:%.*]] = icmp eq i32 [[INDVARS_IV_NEXT]], [[N]]
|
|
; CHECK-NEXT: br i1 [[EXITCOND]], label [[FOR_COND_CLEANUP_LOOPEXIT]], label [[FOR_BODY]]
|
|
;
|
|
entry:
|
|
%cmp31 = icmp sgt i32 %n, 0
|
|
br i1 %cmp31, label %for.body.preheader, label %for.cond.cleanup
|
|
|
|
for.body.preheader:
|
|
br label %for.body
|
|
|
|
for.cond.cleanup.loopexit:
|
|
br label %for.cond.cleanup
|
|
|
|
for.cond.cleanup:
|
|
%sum.0.lcssa = phi i32 [ 0, %entry ], [ %add16, %for.cond.cleanup.loopexit ]
|
|
ret i32 %sum.0.lcssa
|
|
|
|
for.body:
|
|
%indvars.iv = phi i32 [ 0, %for.body.preheader ], [ %indvars.iv.next, %for.body ]
|
|
%sum.032 = phi i32 [ 0, %for.body.preheader ], [ %add16, %for.body ]
|
|
%t4 = shl nsw i32 %indvars.iv, 1
|
|
%t5 = add nsw i32 %t4, 0
|
|
%arrayidx = getelementptr inbounds i32, ptr %g, i32 %t5
|
|
%t6 = load i32, ptr %arrayidx, align 4
|
|
%add1 = add nsw i32 %t6, %sum.032
|
|
%t7 = add nsw i32 %t4, 1
|
|
%arrayidx5 = getelementptr inbounds i32, ptr %g, i32 %t7
|
|
%t8 = load i32, ptr %arrayidx5, align 4
|
|
%add6 = add nsw i32 %add1, %t8
|
|
%t9 = add nsw i32 %t4, %y
|
|
%arrayidx10 = getelementptr inbounds i32, ptr %g, i32 %t9
|
|
%t10 = load i32, ptr %arrayidx10, align 4
|
|
%add11 = add nsw i32 %add6, %t10
|
|
%t11 = add nsw i32 %t4, %z
|
|
%arrayidx15 = getelementptr inbounds i32, ptr %g, i32 %t11
|
|
%t12 = load i32, ptr %arrayidx15, align 4
|
|
%add16 = add nsw i32 %add11, %t12
|
|
%indvars.iv.next = add nuw nsw i32 %indvars.iv, 1
|
|
%exitcond = icmp eq i32 %indvars.iv.next , %n
|
|
br i1 %exitcond, label %for.cond.cleanup.loopexit, label %for.body
|
|
}
|