shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions 6 Packages Projects Releases Wiki Activity
llvm-project/llvm/test/CodeGen/AArch64/machine-licm-sub-loop.ll
Luke Lau 795a115d19
[RegAlloc] Remove default restriction on non-trivial rematerialization (#159211) 
In the register allocator we define non-trivial rematerialization as the
rematerlization of an instruction with virtual register uses.

We have been able to perform non-trivial rematerialization for a while,
but it has been prevented by default unless specifically overriden by
the target in `TargetTransformInfo::isReMaterializableImpl`. The
original reasoning for this given by the comment in the default
implementation is because we might increase a live range of the virtual
register, but we don't actually do this.
LiveRangeEdit::allUsesAvailableAt makes sure that we only rematerialize
instructions whose virtual registers are already live at the use sites.

https://reviews.llvm.org/D106408 had originally tried to remove this
restriction but it was reverted after some performance regressions were
reported. We think it is likely that the regressions were caused by the
fact that the old isTriviallyReMaterializable API sometimes returned
true for non-trivial rematerializations.

However https://github.com/llvm/llvm-project/pull/160377 recently split
the API out into a separate non-trivial and trivial version and updated
the call-sites accordingly, and
https://github.com/llvm/llvm-project/pull/160709 and #159180 fixed
heuristics which weren't accounting for the difference between
non-trivial and trivial.

With these fixes in place, this patch proposes to again allow
non-trivial rematerialization by default which reduces a significant
amount of spills and reloads across various targets.

For llvm-test-suite built with -O3 -flto, we get the following geomean
reduction in reloads:

- arm64-apple-darwin: 11.6%
- riscv64-linux-gnu: 8.1%
- x86_64-linux-gnu: 6.5%
2025-10-04 22:50:44 +00:00

159 lines
7.1 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 2
; RUN: llc -mtriple aarch64-none-linux-gnu < %s | FileCheck %s
define void @foo(i32 noundef %limit, ptr %out, ptr %y) {
; CHECK-LABEL: foo:
; CHECK: // %bb.0: // %entry
; CHECK-NEXT: // kill: def $w0 killed $w0 def $x0
; CHECK-NEXT: cmp w0, #1
; CHECK-NEXT: b.lt .LBB0_10
; CHECK-NEXT: // %bb.1: // %for.cond1.preheader.us.preheader
; CHECK-NEXT: mov w10, w0
; CHECK-NEXT: ubfiz x11, x0, #2, #32
; CHECK-NEXT: mov x8, xzr
; CHECK-NEXT: mov x9, xzr
; CHECK-NEXT: and x12, x10, #0xfffffff0
; CHECK-NEXT: add x13, x1, #32
; CHECK-NEXT: b .LBB0_3
; CHECK-NEXT: .LBB0_2: // %for.cond1.for.cond.cleanup3_crit_edge.us
; CHECK-NEXT: // in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: add x9, x9, #1
; CHECK-NEXT: add x13, x13, x11
; CHECK-NEXT: add x8, x8, x10
; CHECK-NEXT: cmp x9, x10
; CHECK-NEXT: b.eq .LBB0_10
; CHECK-NEXT: .LBB0_3: // %for.cond1.preheader.us
; CHECK-NEXT: // =>This Loop Header: Depth=1
; CHECK-NEXT: // Child Loop BB0_6 Depth 2
; CHECK-NEXT: // Child Loop BB0_9 Depth 2
; CHECK-NEXT: ldrsh w14, [x2, x9, lsl #1]
; CHECK-NEXT: cmp w0, #16
; CHECK-NEXT: b.hs .LBB0_5
; CHECK-NEXT: // %bb.4: // in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: mov x17, xzr
; CHECK-NEXT: b .LBB0_8
; CHECK-NEXT: .LBB0_5: // %vector.ph
; CHECK-NEXT: // in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: dup v0.8h, w14
; CHECK-NEXT: add x15, x2, #16
; CHECK-NEXT: mov x16, x13
; CHECK-NEXT: and x17, x10, #0xfffffff0
; CHECK-NEXT: .LBB0_6: // %vector.body
; CHECK-NEXT: // Parent Loop BB0_3 Depth=1
; CHECK-NEXT: // => This Inner Loop Header: Depth=2
; CHECK-NEXT: ldp q1, q4, [x15, #-16]
; CHECK-NEXT: subs x17, x17, #16
; CHECK-NEXT: ldp q3, q2, [x16, #-32]
; CHECK-NEXT: add x15, x15, #32
; CHECK-NEXT: ldp q6, q5, [x16]
; CHECK-NEXT: smlal2 v2.4s, v0.8h, v1.8h
; CHECK-NEXT: smlal v3.4s, v0.4h, v1.4h
; CHECK-NEXT: smlal2 v5.4s, v0.8h, v4.8h
; CHECK-NEXT: smlal v6.4s, v0.4h, v4.4h
; CHECK-NEXT: stp q3, q2, [x16, #-32]
; CHECK-NEXT: stp q6, q5, [x16], #64
; CHECK-NEXT: b.ne .LBB0_6
; CHECK-NEXT: // %bb.7: // %middle.block
; CHECK-NEXT: // in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: cmp x12, x10
; CHECK-NEXT: and x17, x10, #0xfffffff0
; CHECK-NEXT: b.eq .LBB0_2
; CHECK-NEXT: .LBB0_8: // %for.body4.us.preheader
; CHECK-NEXT: // in Loop: Header=BB0_3 Depth=1
; CHECK-NEXT: add x15, x17, x8
; CHECK-NEXT: add x16, x2, x17, lsl #1
; CHECK-NEXT: sub x17, x10, x17
; CHECK-NEXT: add x15, x1, x15, lsl #2
; CHECK-NEXT: .LBB0_9: // %for.body4.us
; CHECK-NEXT: // Parent Loop BB0_3 Depth=1
; CHECK-NEXT: // => This Inner Loop Header: Depth=2
; CHECK-NEXT: ldrsh w18, [x16], #2
; CHECK-NEXT: ldr w3, [x15]
; CHECK-NEXT: subs x17, x17, #1
; CHECK-NEXT: madd w18, w18, w14, w3
; CHECK-NEXT: str w18, [x15], #4
; CHECK-NEXT: b.ne .LBB0_9
; CHECK-NEXT: b .LBB0_2
; CHECK-NEXT: .LBB0_10: // %for.cond.cleanup
; CHECK-NEXT: ret
entry:
%cmp26 = icmp sgt i32 %limit, 0
br i1 %cmp26, label %for.cond1.preheader.us.preheader, label %for.cond.cleanup
for.cond1.preheader.us.preheader: ; preds = %entry
%0 = zext i32 %limit to i64
%wide.trip.count34 = zext i32 %limit to i64
%min.iters.check = icmp ult i32 %limit, 16
%n.vec = and i64 %wide.trip.count34, 4294967280
%cmp.n = icmp eq i64 %n.vec, %wide.trip.count34
br label %for.cond1.preheader.us
for.cond1.preheader.us: ; preds = %for.cond1.preheader.us.preheader, %for.cond1.for.cond.cleanup3_crit_edge.us
%indvars.iv30 = phi i64 [ 0, %for.cond1.preheader.us.preheader ], [ %indvars.iv.next31, %for.cond1.for.cond.cleanup3_crit_edge.us ]
%arrayidx.us = getelementptr inbounds i16, ptr %y, i64 %indvars.iv30
%1 = load i16, ptr %arrayidx.us, align 2
%conv.us = sext i16 %1 to i32
%2 = mul nsw i64 %indvars.iv30, %0
br i1 %min.iters.check, label %for.body4.us.preheader, label %vector.ph
vector.ph: ; preds = %for.cond1.preheader.us
%broadcast.splatinsert = insertelement <8 x i32> poison, i32 %conv.us, i64 0
%broadcast.splat = shufflevector <8 x i32> %broadcast.splatinsert, <8 x i32> poison, <8 x i32> zeroinitializer
%broadcast.splatinsert37 = insertelement <8 x i32> poison, i32 %conv.us, i64 0
%broadcast.splat38 = shufflevector <8 x i32> %broadcast.splatinsert37, <8 x i32> poison, <8 x i32> zeroinitializer
br label %vector.body
vector.body: ; preds = %vector.body, %vector.ph
%index = phi i64 [ 0, %vector.ph ], [ %index.next, %vector.body ]
%3 = getelementptr inbounds i16, ptr %y, i64 %index
%wide.load = load <8 x i16>, ptr %3, align 2
%4 = getelementptr inbounds i16, ptr %3, i64 8
%wide.load36 = load <8 x i16>, ptr %4, align 2
%5 = sext <8 x i16> %wide.load to <8 x i32>
%6 = sext <8 x i16> %wide.load36 to <8 x i32>
%7 = mul nsw <8 x i32> %broadcast.splat, %5
%8 = mul nsw <8 x i32> %broadcast.splat38, %6
%9 = add nuw nsw i64 %index, %2
%10 = getelementptr inbounds i32, ptr %out, i64 %9
%wide.load39 = load <8 x i32>, ptr %10, align 4
%11 = getelementptr inbounds i32, ptr %10, i64 8
%wide.load40 = load <8 x i32>, ptr %11, align 4
%12 = add nsw <8 x i32> %7, %wide.load39
%13 = add nsw <8 x i32> %8, %wide.load40
store <8 x i32> %12, ptr %10, align 4
store <8 x i32> %13, ptr %11, align 4
%index.next = add nuw i64 %index, 16
%14 = icmp eq i64 %index.next, %n.vec
br i1 %14, label %middle.block, label %vector.body
middle.block: ; preds = %vector.body
br i1 %cmp.n, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us.preheader
for.body4.us.preheader: ; preds = %for.cond1.preheader.us, %middle.block
%indvars.iv.ph = phi i64 [ 0, %for.cond1.preheader.us ], [ %n.vec, %middle.block ]
br label %for.body4.us
for.body4.us: ; preds = %for.body4.us.preheader, %for.body4.us
%indvars.iv = phi i64 [ %indvars.iv.next, %for.body4.us ], [ %indvars.iv.ph, %for.body4.us.preheader ]
%arrayidx6.us = getelementptr inbounds i16, ptr %y, i64 %indvars.iv
%15 = load i16, ptr %arrayidx6.us, align 2
%conv7.us = sext i16 %15 to i32
%mul.us = mul nsw i32 %conv7.us, %conv.us
%16 = add nuw nsw i64 %indvars.iv, %2
%arrayidx10.us = getelementptr inbounds i32, ptr %out, i64 %16
%17 = load i32, ptr %arrayidx10.us, align 4
%add11.us = add nsw i32 %mul.us, %17
store i32 %add11.us, ptr %arrayidx10.us, align 4
%indvars.iv.next = add nuw nsw i64 %indvars.iv, 1
%exitcond.not = icmp eq i64 %indvars.iv.next, %wide.trip.count34
br i1 %exitcond.not, label %for.cond1.for.cond.cleanup3_crit_edge.us, label %for.body4.us
for.cond1.for.cond.cleanup3_crit_edge.us: ; preds = %for.body4.us, %middle.block
%indvars.iv.next31 = add nuw nsw i64 %indvars.iv30, 1
%exitcond35.not = icmp eq i64 %indvars.iv.next31, %wide.trip.count34
br i1 %exitcond35.not, label %for.cond.cleanup, label %for.cond1.preheader.us
for.cond.cleanup: ; preds = %for.cond1.for.cond.cleanup3_crit_edge.us, %entry
ret void
}
Reference in New Issue View Git Blame Copy Permalink