ffb2af3ed6
LSR uses SCEVExpander to generate induction formulas. The expander internally tries to reuse existing IR expressions. To do that, it needs to strip any poison generating flags (nsw, nuw, exact, nneg, etc..) which may not be valid for the newly added users. This is conservatively correct, but has the effect that LSR will strip nneg flags on zext instructions involved in trip counts in loop preheaders. To avoid this, this patch adjusts the expanded to reinfer the flags on the CSE candidate if legal for all possible users. This should fix the regression reported in https://github.com/llvm/llvm-project/issues/71200. This should arguably be done inside canReuseInstruction instead, but doing it outside is more conservative compile time wise. Both canReuseInstruction and isGuaranteedNotToBePoison walk operand lists, so right now we are performing work which is roughly O(N^2) in the size of the operand graph. We should fix that before making the per operand step more expensive. My tenative plan is to land this, and then rework the code to sink the logic into more core interfaces.
118 lines
5.7 KiB
LLVM
118 lines
5.7 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py
|
|
; RUN: opt -S -passes=irce,verify -verify-loop-info < %s 2>&1 | FileCheck %s
|
|
; RUN: opt -S -verify-loop-info -passes=irce,verify < %s 2>&1 | FileCheck %s
|
|
|
|
define i32 @test_01(i32 %A, i64 %Len, ptr %array) {
|
|
; CHECK-LABEL: @test_01(
|
|
; CHECK-NEXT: preheader:
|
|
; CHECK-NEXT: [[TRIPCHECK:%.*]] = icmp sgt i64 [[LEN:%.*]], 2
|
|
; CHECK-NEXT: br i1 [[TRIPCHECK]], label [[LOOP_PREHEADER:%.*]], label [[ZERO:%.*]]
|
|
; CHECK: loop.preheader:
|
|
; CHECK-NEXT: [[TMP0:%.*]] = zext i32 [[A:%.*]] to i64
|
|
; CHECK-NEXT: [[TMP1:%.*]] = add nuw nsw i64 [[TMP0]], 1
|
|
; CHECK-NEXT: [[SMIN:%.*]] = call i64 @llvm.smin.i64(i64 [[LEN]], i64 0)
|
|
; CHECK-NEXT: [[TMP2:%.*]] = sub i64 [[LEN]], [[SMIN]]
|
|
; CHECK-NEXT: [[UMIN:%.*]] = call i64 @llvm.umin.i64(i64 [[TMP2]], i64 [[TMP1]])
|
|
; CHECK-NEXT: [[EXIT_MAINLOOP_AT:%.*]] = call i64 @llvm.umax.i64(i64 [[UMIN]], i64 1)
|
|
; CHECK-NEXT: [[TMP3:%.*]] = icmp ult i64 1, [[EXIT_MAINLOOP_AT]]
|
|
; CHECK-NEXT: br i1 [[TMP3]], label [[LOOP_PREHEADER2:%.*]], label [[MAIN_PSEUDO_EXIT:%.*]]
|
|
; CHECK: loop.preheader2:
|
|
; CHECK-NEXT: br label [[LOOP:%.*]]
|
|
; CHECK: loop:
|
|
; CHECK-NEXT: [[INDVAR:%.*]] = phi i64 [ [[INDVAR_NEXT:%.*]], [[LATCH:%.*]] ], [ 1, [[LOOP_PREHEADER2]] ]
|
|
; CHECK-NEXT: [[TMP4:%.*]] = icmp ult i64 [[INDVAR]], [[LEN]]
|
|
; CHECK-NEXT: br i1 true, label [[GUARDED:%.*]], label [[DEOPT_LOOPEXIT3:%.*]]
|
|
; CHECK: guarded:
|
|
; CHECK-NEXT: [[ADDR:%.*]] = getelementptr inbounds i32, ptr [[ARRAY:%.*]], i64 [[INDVAR]]
|
|
; CHECK-NEXT: [[RES:%.*]] = load i32, ptr [[ADDR]], align 4
|
|
; CHECK-NEXT: [[CMP:%.*]] = icmp ne i32 [[RES]], 0
|
|
; CHECK-NEXT: br i1 [[CMP]], label [[LATCH]], label [[ZERO_LOOPEXIT_LOOPEXIT4:%.*]]
|
|
; CHECK: latch:
|
|
; CHECK-NEXT: [[INDVAR_NEXT]] = add nuw nsw i64 [[INDVAR]], 2
|
|
; CHECK-NEXT: [[RES2:%.*]] = mul i32 [[RES]], 3
|
|
; CHECK-NEXT: [[TMP5:%.*]] = zext i32 [[A]] to i64
|
|
; CHECK-NEXT: [[CMP2:%.*]] = icmp ugt i64 [[INDVAR_NEXT]], [[TMP5]]
|
|
; CHECK-NEXT: [[TMP6:%.*]] = icmp ult i64 [[INDVAR_NEXT]], [[EXIT_MAINLOOP_AT]]
|
|
; CHECK-NEXT: [[TMP7:%.*]] = xor i1 [[TMP6]], true
|
|
; CHECK-NEXT: br i1 [[TMP7]], label [[MAIN_EXIT_SELECTOR:%.*]], label [[LOOP]]
|
|
; CHECK: main.exit.selector:
|
|
; CHECK-NEXT: [[INDVAR_NEXT_LCSSA:%.*]] = phi i64 [ [[INDVAR_NEXT]], [[LATCH]] ]
|
|
; CHECK-NEXT: [[RES2_LCSSA1:%.*]] = phi i32 [ [[RES2]], [[LATCH]] ]
|
|
; CHECK-NEXT: [[TMP8:%.*]] = icmp ult i64 [[INDVAR_NEXT_LCSSA]], [[TMP1]]
|
|
; CHECK-NEXT: br i1 [[TMP8]], label [[MAIN_PSEUDO_EXIT]], label [[LOOPEXIT:%.*]]
|
|
; CHECK: main.pseudo.exit:
|
|
; CHECK-NEXT: [[INDVAR_COPY:%.*]] = phi i64 [ 1, [[LOOP_PREHEADER]] ], [ [[INDVAR_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
|
|
; CHECK-NEXT: [[INDVAR_END:%.*]] = phi i64 [ 1, [[LOOP_PREHEADER]] ], [ [[INDVAR_NEXT_LCSSA]], [[MAIN_EXIT_SELECTOR]] ]
|
|
; CHECK-NEXT: br label [[POSTLOOP:%.*]]
|
|
; CHECK: loopexit.loopexit:
|
|
; CHECK-NEXT: [[RES2_LCSSA_PH:%.*]] = phi i32 [ [[RES2_POSTLOOP:%.*]], [[LATCH_POSTLOOP:%.*]] ]
|
|
; CHECK-NEXT: br label [[LOOPEXIT]]
|
|
; CHECK: loopexit:
|
|
; CHECK-NEXT: [[RES2_LCSSA:%.*]] = phi i32 [ [[RES2_LCSSA1]], [[MAIN_EXIT_SELECTOR]] ], [ [[RES2_LCSSA_PH]], [[LOOPEXIT_LOOPEXIT:%.*]] ]
|
|
; CHECK-NEXT: ret i32 [[RES2_LCSSA]]
|
|
; CHECK: zero.loopexit.loopexit:
|
|
; CHECK-NEXT: br label [[ZERO_LOOPEXIT:%.*]]
|
|
; CHECK: zero.loopexit.loopexit4:
|
|
; CHECK-NEXT: br label [[ZERO_LOOPEXIT]]
|
|
; CHECK: zero.loopexit:
|
|
; CHECK-NEXT: br label [[ZERO]]
|
|
; CHECK: zero:
|
|
; CHECK-NEXT: ret i32 0
|
|
; CHECK: deopt.loopexit:
|
|
; CHECK-NEXT: br label [[DEOPT:%.*]]
|
|
; CHECK: deopt.loopexit3:
|
|
; CHECK-NEXT: br label [[DEOPT]]
|
|
; CHECK: deopt:
|
|
; CHECK-NEXT: ret i32 1
|
|
; CHECK: postloop:
|
|
; CHECK-NEXT: br label [[LOOP_POSTLOOP:%.*]]
|
|
; CHECK: loop.postloop:
|
|
; CHECK-NEXT: [[INDVAR_POSTLOOP:%.*]] = phi i64 [ [[INDVAR_NEXT_POSTLOOP:%.*]], [[LATCH_POSTLOOP]] ], [ [[INDVAR_COPY]], [[POSTLOOP]] ]
|
|
; CHECK-NEXT: [[TMP9:%.*]] = icmp ult i64 [[INDVAR_POSTLOOP]], [[LEN]]
|
|
; CHECK-NEXT: br i1 [[TMP9]], label [[GUARDED_POSTLOOP:%.*]], label [[DEOPT_LOOPEXIT:%.*]]
|
|
; CHECK: guarded.postloop:
|
|
; CHECK-NEXT: [[ADDR_POSTLOOP:%.*]] = getelementptr inbounds i32, ptr [[ARRAY]], i64 [[INDVAR_POSTLOOP]]
|
|
; CHECK-NEXT: [[RES_POSTLOOP:%.*]] = load i32, ptr [[ADDR_POSTLOOP]], align 4
|
|
; CHECK-NEXT: [[CMP_POSTLOOP:%.*]] = icmp ne i32 [[RES_POSTLOOP]], 0
|
|
; CHECK-NEXT: br i1 [[CMP_POSTLOOP]], label [[LATCH_POSTLOOP]], label [[ZERO_LOOPEXIT_LOOPEXIT:%.*]]
|
|
; CHECK: latch.postloop:
|
|
; CHECK-NEXT: [[INDVAR_NEXT_POSTLOOP]] = add nuw nsw i64 [[INDVAR_POSTLOOP]], 2
|
|
; CHECK-NEXT: [[RES2_POSTLOOP]] = mul i32 [[RES_POSTLOOP]], 3
|
|
; CHECK-NEXT: [[TMP10:%.*]] = zext i32 [[A]] to i64
|
|
; CHECK-NEXT: [[CMP2_POSTLOOP:%.*]] = icmp ugt i64 [[INDVAR_NEXT_POSTLOOP]], [[TMP10]]
|
|
; CHECK-NEXT: br i1 [[CMP2_POSTLOOP]], label [[LOOPEXIT_LOOPEXIT]], label [[LOOP_POSTLOOP]], !llvm.loop [[LOOP0:![0-9]+]], !loop_constrainer.loop.clone !5
|
|
;
|
|
preheader:
|
|
%tripcheck = icmp sgt i64 %Len, 2
|
|
br i1 %tripcheck, label %loop, label %zero
|
|
|
|
loop:
|
|
%indvar = phi i64 [ 1, %preheader ], [ %indvar.next, %latch ]
|
|
%0 = icmp ult i64 %indvar, %Len
|
|
br i1 %0, label %guarded, label %deopt
|
|
|
|
guarded:
|
|
%addr = getelementptr inbounds i32, ptr %array, i64 %indvar
|
|
%res = load i32, ptr %addr
|
|
%cmp = icmp eq i32 %res, 0
|
|
br i1 %cmp, label %zero, label %latch
|
|
|
|
latch:
|
|
%indvar.next = add nuw nsw i64 %indvar, 2
|
|
%res2 = mul i32 %res, 3
|
|
; NOTE: this is loop invariant value, but not loop invariant instruction!
|
|
%1 = zext i32 %A to i64
|
|
%cmp2 = icmp ugt i64 %indvar.next, %1
|
|
br i1 %cmp2, label %loopexit, label %loop
|
|
|
|
loopexit:
|
|
ret i32 %res2
|
|
|
|
zero:
|
|
ret i32 0
|
|
|
|
deopt:
|
|
ret i32 1
|
|
|
|
}
|