a8e1311a1c
This is intended to solve a problem with lowering atomics in OpenMP and C++ common to AMDGPU and NVPTX. In OpenCL and CUDA, it is undefined behavior for an atomic instruction to modify an object in thread private memory. In OpenMP, it is defined. Correspondingly, the hardware does not handle this correctly. For AMDGPU, 32-bit atomics work and 64-bit atomics are silently dropped. We therefore need to codegen this by inserting a runtime address space check, performing the private case without atomics, and fallback to issuing the real atomic otherwise. This metadata allows us to avoid this extra check and branch. Handle this by introducing metadata intended to be applied to atomicrmw, indicating they cannot access the forbidden address space.
402 lines
9.5 KiB
LLVM
402 lines
9.5 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --check-globals
|
|
; RUN: opt < %s -passes=simplifycfg -simplifycfg-require-and-preserve-domtree=1 -hoist-common-insts=true -S | FileCheck %s
|
|
|
|
define void @hoist_range(i1 %c, ptr %p) {
|
|
; CHECK-LABEL: @hoist_range(
|
|
; CHECK-NEXT: if:
|
|
; CHECK-NEXT: [[T:%.*]] = load i8, ptr [[P:%.*]], align 1, !range [[RNG0:![0-9]+]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
if:
|
|
br i1 %c, label %then, label %else
|
|
then:
|
|
%t = load i8, ptr %p, !range !0
|
|
br label %out
|
|
else:
|
|
%e = load i8, ptr %p, !range !1
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_range_switch(i64 %i, ptr %p) {
|
|
; CHECK-LABEL: @hoist_range_switch(
|
|
; CHECK-NEXT: out:
|
|
; CHECK-NEXT: [[T:%.*]] = load i8, ptr [[P:%.*]], align 1, !range [[RNG1:![0-9]+]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
switch i64 %i, label %bb0 [
|
|
i64 1, label %bb1
|
|
i64 2, label %bb2
|
|
]
|
|
bb0:
|
|
%t = load i8, ptr %p, !range !0
|
|
br label %out
|
|
bb1:
|
|
%e = load i8, ptr %p, !range !1
|
|
br label %out
|
|
bb2:
|
|
%f = load i8, ptr %p, !range !3
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_both_noundef(i1 %c, ptr %p) {
|
|
; CHECK-LABEL: @hoist_both_noundef(
|
|
; CHECK-NEXT: if:
|
|
; CHECK-NEXT: [[T:%.*]] = load i8, ptr [[P:%.*]], align 1, !noundef [[META2:![0-9]+]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
if:
|
|
br i1 %c, label %then, label %else
|
|
|
|
then:
|
|
%t = load i8, ptr %p, !noundef !2
|
|
br label %out
|
|
|
|
else:
|
|
%e = load i8, ptr %p, !noundef !2
|
|
br label %out
|
|
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
|
|
define void @hoist_both_noundef_switch(i64 %i, ptr %p) {
|
|
; CHECK-LABEL: @hoist_both_noundef_switch(
|
|
; CHECK-NEXT: out:
|
|
; CHECK-NEXT: [[T:%.*]] = load i8, ptr [[P:%.*]], align 1, !noundef [[META2]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
switch i64 %i, label %bb0 [
|
|
i64 1, label %bb1
|
|
i64 2, label %bb2
|
|
]
|
|
bb0:
|
|
%t = load i8, ptr %p, !noundef !2
|
|
br label %out
|
|
bb1:
|
|
%e = load i8, ptr %p, !noundef !2
|
|
br label %out
|
|
bb2:
|
|
%f = load i8, ptr %p, !noundef !2
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_one_noundef(i1 %c, ptr %p) {
|
|
; CHECK-LABEL: @hoist_one_noundef(
|
|
; CHECK-NEXT: if:
|
|
; CHECK-NEXT: [[T:%.*]] = load i8, ptr [[P:%.*]], align 1
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
if:
|
|
br i1 %c, label %then, label %else
|
|
|
|
then:
|
|
%t = load i8, ptr %p, !noundef !2
|
|
br label %out
|
|
|
|
else:
|
|
%e = load i8, ptr %p
|
|
br label %out
|
|
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_one_noundef_switch(i64 %i, ptr %p) {
|
|
; CHECK-LABEL: @hoist_one_noundef_switch(
|
|
; CHECK-NEXT: out:
|
|
; CHECK-NEXT: [[T:%.*]] = load i8, ptr [[P:%.*]], align 1
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
switch i64 %i, label %bb0 [
|
|
i64 1, label %bb1
|
|
i64 2, label %bb2
|
|
]
|
|
bb0:
|
|
%t = load i8, ptr %p, !noundef !2
|
|
br label %out
|
|
bb1:
|
|
%e = load i8, ptr %p
|
|
br label %out
|
|
bb2:
|
|
%f = load i8, ptr %p, !noundef !2
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_dereferenceable(i1 %c, ptr %p) {
|
|
; CHECK-LABEL: @hoist_dereferenceable(
|
|
; CHECK-NEXT: if:
|
|
; CHECK-NEXT: [[T:%.*]] = load ptr, ptr [[P:%.*]], align 8, !dereferenceable [[META3:![0-9]+]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
if:
|
|
br i1 %c, label %then, label %else
|
|
then:
|
|
%t = load ptr, ptr %p, !dereferenceable !{i64 10}
|
|
br label %out
|
|
else:
|
|
%e = load ptr, ptr %p, !dereferenceable !{i64 20}
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_dereferenceable_switch(i64 %i, ptr %p) {
|
|
; CHECK-LABEL: @hoist_dereferenceable_switch(
|
|
; CHECK-NEXT: out:
|
|
; CHECK-NEXT: [[T:%.*]] = load ptr, ptr [[P:%.*]], align 8, !dereferenceable [[META3]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
switch i64 %i, label %bb0 [
|
|
i64 1, label %bb1
|
|
i64 2, label %bb2
|
|
]
|
|
bb0:
|
|
%t = load ptr, ptr %p, !dereferenceable !{i64 10}
|
|
br label %out
|
|
bb1:
|
|
%e = load ptr, ptr %p, !dereferenceable !{i64 20}
|
|
br label %out
|
|
bb2:
|
|
%f = load ptr, ptr %p, !dereferenceable !{i64 30}
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_dereferenceable_or_null(i1 %c, ptr %p) {
|
|
; CHECK-LABEL: @hoist_dereferenceable_or_null(
|
|
; CHECK-NEXT: if:
|
|
; CHECK-NEXT: [[T:%.*]] = load ptr, ptr [[P:%.*]], align 8, !dereferenceable_or_null [[META3]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
if:
|
|
br i1 %c, label %then, label %else
|
|
then:
|
|
%t = load ptr, ptr %p, !dereferenceable_or_null !{i64 20}
|
|
br label %out
|
|
else:
|
|
%e = load ptr, ptr %p, !dereferenceable_or_null !{i64 10}
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_dereferenceable_or_null_switch(i64 %i, ptr %p) {
|
|
; CHECK-LABEL: @hoist_dereferenceable_or_null_switch(
|
|
; CHECK-NEXT: out:
|
|
; CHECK-NEXT: [[T:%.*]] = load ptr, ptr [[P:%.*]], align 8, !dereferenceable_or_null [[META3]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
switch i64 %i, label %bb0 [
|
|
i64 1, label %bb1
|
|
i64 2, label %bb2
|
|
]
|
|
bb0:
|
|
%t = load ptr, ptr %p, !dereferenceable_or_null !{i64 20}
|
|
br label %out
|
|
bb1:
|
|
%e = load ptr, ptr %p, !dereferenceable_or_null !{i64 10}
|
|
br label %out
|
|
bb2:
|
|
%f = load ptr, ptr %p, !dereferenceable_or_null !{i64 30}
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
; !range violation only returns poison, and is thus safe to speculate.
|
|
define i32 @speculate_range(i1 %c, ptr dereferenceable(8) align 8 %p) {
|
|
; CHECK-LABEL: @speculate_range(
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[V:%.*]] = load i32, ptr [[P:%.*]], align 4, !range [[RNG4:![0-9]+]]
|
|
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[C:%.*]], i32 [[V]], i32 0
|
|
; CHECK-NEXT: ret i32 [[SPEC_SELECT]]
|
|
;
|
|
entry:
|
|
br i1 %c, label %if, label %join
|
|
|
|
if:
|
|
%v = load i32, ptr %p, !range !{i32 0, i32 10}
|
|
br label %join
|
|
|
|
join:
|
|
%phi = phi i32 [ %v, %if ], [ 0, %entry ]
|
|
ret i32 %phi
|
|
}
|
|
|
|
; !nonnull is safe to speculate, but !noundef is not, as the latter causes
|
|
; immediate undefined behavior.
|
|
define ptr @speculate_nonnull(i1 %c, ptr dereferenceable(8) align 8 %p) {
|
|
; CHECK-LABEL: @speculate_nonnull(
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[V:%.*]] = load ptr, ptr [[P:%.*]], align 8, !nonnull [[META2]]
|
|
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[C:%.*]], ptr [[V]], ptr null
|
|
; CHECK-NEXT: ret ptr [[SPEC_SELECT]]
|
|
;
|
|
entry:
|
|
br i1 %c, label %if, label %join
|
|
|
|
if:
|
|
%v = load ptr, ptr %p, !nonnull !{}, !noundef !{}
|
|
br label %join
|
|
|
|
join:
|
|
%phi = phi ptr [ %v, %if ], [ null, %entry ]
|
|
ret ptr %phi
|
|
}
|
|
|
|
; !align is safe to speculate, but !dereferenceable is not, as the latter causes
|
|
; immediate undefined behavior.
|
|
define ptr @speculate_align(i1 %c, ptr dereferenceable(8) align 8 %p) {
|
|
; CHECK-LABEL: @speculate_align(
|
|
; CHECK-NEXT: entry:
|
|
; CHECK-NEXT: [[V:%.*]] = load ptr, ptr [[P:%.*]], align 8, !align [[META5:![0-9]+]]
|
|
; CHECK-NEXT: [[SPEC_SELECT:%.*]] = select i1 [[C:%.*]], ptr [[V]], ptr null
|
|
; CHECK-NEXT: ret ptr [[SPEC_SELECT]]
|
|
;
|
|
entry:
|
|
br i1 %c, label %if, label %join
|
|
|
|
if:
|
|
%v = load ptr, ptr %p, !align !{i64 4}, !dereferenceable !{i64 4}
|
|
br label %join
|
|
|
|
join:
|
|
%phi = phi ptr [ %v, %if ], [ null, %entry ]
|
|
ret ptr %phi
|
|
}
|
|
|
|
define void @hoist_fpmath(i1 %c, double %x) {
|
|
; CHECK-LABEL: @hoist_fpmath(
|
|
; CHECK-NEXT: if:
|
|
; CHECK-NEXT: [[T:%.*]] = fadd double [[X:%.*]], 1.000000e+00, !fpmath [[META6:![0-9]+]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
if:
|
|
br i1 %c, label %then, label %else
|
|
then:
|
|
%t = fadd double %x, 1.0, !fpmath !{ float 2.5 }
|
|
br label %out
|
|
else:
|
|
%e = fadd double %x, 1.0, !fpmath !{ float 5.0 }
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_fpmath_switch(i64 %i, double %x) {
|
|
; CHECK-LABEL: @hoist_fpmath_switch(
|
|
; CHECK-NEXT: out:
|
|
; CHECK-NEXT: [[T:%.*]] = fadd double [[X:%.*]], 1.000000e+00, !fpmath [[META6]]
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
switch i64 %i, label %bb0 [
|
|
i64 1, label %bb1
|
|
i64 2, label %bb2
|
|
]
|
|
bb0:
|
|
%t = fadd double %x, 1.0, !fpmath !{ float 2.5 }
|
|
br label %out
|
|
bb1:
|
|
%e = fadd double %x, 1.0, !fpmath !{ float 5.0 }
|
|
br label %out
|
|
bb2:
|
|
%f = fadd double %x, 1.0, !fpmath !{ float 7.5 }
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_noalias_addrspace_both(i1 %c, ptr %p, i64 %val) {
|
|
; CHECK-LABEL: @hoist_noalias_addrspace_both(
|
|
; CHECK-NEXT: if:
|
|
; CHECK-NEXT: [[T:%.*]] = atomicrmw add ptr [[P:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
if:
|
|
br i1 %c, label %then, label %else
|
|
|
|
then:
|
|
%t = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !4
|
|
br label %out
|
|
|
|
else:
|
|
%e = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !4
|
|
br label %out
|
|
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_noalias_addrspace_one(i1 %c, ptr %p, i64 %val) {
|
|
; CHECK-LABEL: @hoist_noalias_addrspace_one(
|
|
; CHECK-NEXT: if:
|
|
; CHECK-NEXT: [[T:%.*]] = atomicrmw add ptr [[P:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
if:
|
|
br i1 %c, label %then, label %else
|
|
|
|
then:
|
|
%t = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !4
|
|
br label %out
|
|
|
|
else:
|
|
%e = atomicrmw add ptr %p, i64 %val seq_cst
|
|
br label %out
|
|
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
define void @hoist_noalias_addrspace_switch(i64 %i, ptr %p, i64 %val) {
|
|
; CHECK-LABEL: @hoist_noalias_addrspace_switch(
|
|
; CHECK-NEXT: out:
|
|
; CHECK-NEXT: [[T:%.*]] = atomicrmw add ptr [[P:%.*]], i64 [[VAL:%.*]] seq_cst, align 8
|
|
; CHECK-NEXT: ret void
|
|
;
|
|
switch i64 %i, label %bb0 [
|
|
i64 1, label %bb1
|
|
i64 2, label %bb2
|
|
]
|
|
bb0:
|
|
%t = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !4
|
|
br label %out
|
|
bb1:
|
|
%e = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !5
|
|
br label %out
|
|
bb2:
|
|
%f = atomicrmw add ptr %p, i64 %val seq_cst, !noalias.addrspace !6
|
|
br label %out
|
|
out:
|
|
ret void
|
|
}
|
|
|
|
|
|
!0 = !{ i8 0, i8 1 }
|
|
!1 = !{ i8 3, i8 5 }
|
|
!2 = !{}
|
|
!3 = !{ i8 7, i8 9 }
|
|
!4 = !{i32 5, i32 6}
|
|
!5 = !{i32 5, i32 7}
|
|
!6 = !{i32 4, i32 8}
|
|
|
|
;.
|
|
; CHECK: [[RNG0]] = !{i8 0, i8 1, i8 3, i8 5}
|
|
; CHECK: [[RNG1]] = !{i8 0, i8 1, i8 3, i8 5, i8 7, i8 9}
|
|
; CHECK: [[META2]] = !{}
|
|
; CHECK: [[META3]] = !{i64 10}
|
|
; CHECK: [[RNG4]] = !{i32 0, i32 10}
|
|
; CHECK: [[META5]] = !{i64 4}
|
|
; CHECK: [[META6]] = !{float 2.500000e+00}
|
|
;.
|