shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/llvm/test/Transforms/LoopVectorize/multiple-result-intrinsics.ll
Benjamin Maxwell 89e7f4d31b
[LV] Teach the vectorizer to cost and vectorize modf and sincospi intrinsics (#129064) 
Follow on to #128035. It is a small extension to support vectorizing
`llvm.modf.*` and `llvm.sincospi.*` too.

This renames the test files from `sincos.ll` ->
`multiple-result-intrinsics.ll` to group together the similar tests
(which make up most of this PR).
2025-02-28 12:56:12 +00:00

331 lines
15 KiB
LLVM
Raw Blame History

; NOTE: Assertions have been autogenerated by utils/update_test_checks.py UTC_ARGS: --filter "(:|sincos|modf|extract|store)" --version 5
; RUN: opt -passes=loop-vectorize -force-vector-interleave=1 -force-vector-width=2 < %s -S -o - | FileCheck %s
define void @sincos_f32(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @sincos_f32(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
; CHECK: [[ENTRY:.*:]]
; CHECK: [[VECTOR_PH:.*:]]
; CHECK: [[VECTOR_BODY:.*:]]
; CHECK: [[TMP3:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> [[WIDE_LOAD:%.*]])
; CHECK: [[TMP4:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 0
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 1
; CHECK: store <2 x float> [[TMP4]], ptr [[TMP7:%.*]], align 4
; CHECK: store <2 x float> [[TMP5]], ptr [[TMP9:%.*]], align 4
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[SCALAR_PH:.*:]]
; CHECK: [[FOR_BODY:.*:]]
; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
; CHECK: [[EXIT:.*:]]
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
%in_val = load float, ptr %arrayidx, align 4
%call = tail call { float, float } @llvm.sincos.f32(float %in_val)
%extract_a = extractvalue { float, float } %call, 0
%extract_b = extractvalue { float, float } %call, 1
%arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
store float %extract_a, ptr %arrayidx2, align 4
%arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
store float %extract_b, ptr %arrayidx4, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %exit, label %for.body
exit:
ret void
}
define void @sincos_f64(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @sincos_f64(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
; CHECK: [[ENTRY:.*:]]
; CHECK: [[VECTOR_PH:.*:]]
; CHECK: [[VECTOR_BODY:.*:]]
; CHECK: [[TMP3:%.*]] = call { <2 x double>, <2 x double> } @llvm.sincos.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
; CHECK: [[TMP4:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 0
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 1
; CHECK: store <2 x double> [[TMP4]], ptr [[TMP7:%.*]], align 8
; CHECK: store <2 x double> [[TMP5]], ptr [[TMP9:%.*]], align 8
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[SCALAR_PH:.*:]]
; CHECK: [[FOR_BODY:.*:]]
; CHECK: [[CALL:%.*]] = tail call { double, double } @llvm.sincos.f64(double [[IN_VAL:%.*]])
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { double, double } [[CALL]], 0
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { double, double } [[CALL]], 1
; CHECK: store double [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 8
; CHECK: store double [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 8
; CHECK: [[EXIT:.*:]]
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds double, ptr %in, i64 %iv
%in_val = load double, ptr %arrayidx, align 8
%call = tail call { double, double } @llvm.sincos.f64(double %in_val)
%extract_a = extractvalue { double, double } %call, 0
%extract_b = extractvalue { double, double } %call, 1
%arrayidx2 = getelementptr inbounds double, ptr %out_a, i64 %iv
store double %extract_a, ptr %arrayidx2, align 8
%arrayidx4 = getelementptr inbounds double, ptr %out_b, i64 %iv
store double %extract_b, ptr %arrayidx4, align 8
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %exit, label %for.body
exit:
ret void
}
define void @predicated_sincos(float %x, ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @predicated_sincos(
; CHECK-SAME: float [[X:%.*]], ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
; CHECK: [[ENTRY:.*:]]
; CHECK: [[VECTOR_BODY1:.*]]:
; CHECK: [[VECTOR_BODY:.*:]]
; CHECK: [[INDEX:%.*]] = phi i64 [ 0, %[[VECTOR_BODY1]] ], [ [[INDEX_NEXT:%.*]], %[[IF_THEN2:.*]] ]
; CHECK: [[TMP4:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincos.v2f32(<2 x float> [[WIDE_LOAD:%.*]])
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP4]], 0
; CHECK: [[TMP6:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP4]], 1
; CHECK: [[TMP7:%.*]] = extractelement <2 x i1> [[TMP3:%.*]], i32 0
; CHECK: br i1 [[TMP7]], label %[[PRED_STORE_IF:.*]], label %[[PRED_STORE_CONTINUE:.*]]
; CHECK: [[PRED_STORE_IF]]:
; CHECK: [[TMP9:%.*]] = extractelement <2 x float> [[TMP5]], i32 0
; CHECK: store float [[TMP9]], ptr [[TMP8:%.*]], align 4
; CHECK: [[TMP11:%.*]] = extractelement <2 x float> [[TMP6]], i32 0
; CHECK: store float [[TMP11]], ptr [[TMP10:%.*]], align 4
; CHECK: br label %[[PRED_STORE_CONTINUE]]
; CHECK: [[PRED_STORE_CONTINUE]]:
; CHECK: [[TMP12:%.*]] = extractelement <2 x i1> [[TMP3]], i32 1
; CHECK: br i1 [[TMP12]], label %[[PRED_STORE_IF1:.*]], label %[[IF_THEN2]]
; CHECK: [[PRED_STORE_IF1]]:
; CHECK: [[TMP15:%.*]] = extractelement <2 x float> [[TMP5]], i32 1
; CHECK: store float [[TMP15]], ptr [[TMP14:%.*]], align 4
; CHECK: [[TMP17:%.*]] = extractelement <2 x float> [[TMP6]], i32 1
; CHECK: store float [[TMP17]], ptr [[TMP16:%.*]], align 4
; CHECK: br label %[[IF_THEN2]]
; CHECK: [[IF_THEN2]]:
; CHECK: [[IF_THEN:.*:]]
; CHECK: [[IF_THEN3:.*:]]
; CHECK: [[IF_THEN4:.*:]]
; CHECK: [[IF_THEN1:.*:]]
; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.sincos.f32(float [[IN_VAL:%.*]])
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
; CHECK: [[IF_MERGE:.*:]]
; CHECK: [[FOR_END:.*:]]
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ %iv.next, %if.merge ], [ 0, %entry ]
%arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
%in_val = load float, ptr %arrayidx, align 4
%if_cond = fcmp olt float %in_val, %x
br i1 %if_cond, label %if.then, label %if.merge
if.then:
%call = tail call { float, float } @llvm.sincos.f32(float %in_val)
%extract_a = extractvalue { float, float } %call, 0
%extract_b = extractvalue { float, float } %call, 1
%arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
store float %extract_a, ptr %arrayidx2, align 4
%arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
store float %extract_b, ptr %arrayidx4, align 4
br label %if.merge
if.merge:
%iv.next = add nuw nsw i64 %iv, 1
%cond = icmp slt i64 %iv.next, 1024
br i1 %cond, label %for.body, label %for.end
for.end:
ret void
}
define void @modf_f32(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @modf_f32(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
; CHECK: [[ENTRY:.*:]]
; CHECK: [[VECTOR_PH:.*:]]
; CHECK: [[VECTOR_BODY:.*:]]
; CHECK: [[TMP3:%.*]] = call { <2 x float>, <2 x float> } @llvm.modf.v2f32(<2 x float> [[WIDE_LOAD:%.*]])
; CHECK: [[TMP4:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 0
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 1
; CHECK: store <2 x float> [[TMP4]], ptr [[TMP7:%.*]], align 4
; CHECK: store <2 x float> [[TMP5]], ptr [[TMP9:%.*]], align 4
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[SCALAR_PH:.*:]]
; CHECK: [[FOR_BODY:.*:]]
; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.modf.f32(float [[IN_VAL:%.*]])
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
; CHECK: [[EXIT:.*:]]
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
%in_val = load float, ptr %arrayidx, align 4
%call = tail call { float, float } @llvm.modf.f32(float %in_val)
%extract_a = extractvalue { float, float } %call, 0
%extract_b = extractvalue { float, float } %call, 1
%arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
store float %extract_a, ptr %arrayidx2, align 4
%arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
store float %extract_b, ptr %arrayidx4, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %exit, label %for.body
exit:
ret void
}
define void @modf_f64(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @modf_f64(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
; CHECK: [[ENTRY:.*:]]
; CHECK: [[VECTOR_PH:.*:]]
; CHECK: [[VECTOR_BODY:.*:]]
; CHECK: [[TMP3:%.*]] = call { <2 x double>, <2 x double> } @llvm.modf.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
; CHECK: [[TMP4:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 0
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 1
; CHECK: store <2 x double> [[TMP4]], ptr [[TMP7:%.*]], align 8
; CHECK: store <2 x double> [[TMP5]], ptr [[TMP9:%.*]], align 8
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[SCALAR_PH:.*:]]
; CHECK: [[FOR_BODY:.*:]]
; CHECK: [[CALL:%.*]] = tail call { double, double } @llvm.modf.f64(double [[IN_VAL:%.*]])
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { double, double } [[CALL]], 0
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { double, double } [[CALL]], 1
; CHECK: store double [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 8
; CHECK: store double [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 8
; CHECK: [[EXIT:.*:]]
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds double, ptr %in, i64 %iv
%in_val = load double, ptr %arrayidx, align 8
%call = tail call { double, double } @llvm.modf.f64(double %in_val)
%extract_a = extractvalue { double, double } %call, 0
%extract_b = extractvalue { double, double } %call, 1
%arrayidx2 = getelementptr inbounds double, ptr %out_a, i64 %iv
store double %extract_a, ptr %arrayidx2, align 8
%arrayidx4 = getelementptr inbounds double, ptr %out_b, i64 %iv
store double %extract_b, ptr %arrayidx4, align 8
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %exit, label %for.body
exit:
ret void
}
define void @sincospi_f32(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @sincospi_f32(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
; CHECK: [[ENTRY:.*:]]
; CHECK: [[VECTOR_PH:.*:]]
; CHECK: [[VECTOR_BODY:.*:]]
; CHECK: [[TMP3:%.*]] = call { <2 x float>, <2 x float> } @llvm.sincospi.v2f32(<2 x float> [[WIDE_LOAD:%.*]])
; CHECK: [[TMP4:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 0
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x float>, <2 x float> } [[TMP3]], 1
; CHECK: store <2 x float> [[TMP4]], ptr [[TMP7:%.*]], align 4
; CHECK: store <2 x float> [[TMP5]], ptr [[TMP9:%.*]], align 4
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[SCALAR_PH:.*:]]
; CHECK: [[FOR_BODY:.*:]]
; CHECK: [[CALL:%.*]] = tail call { float, float } @llvm.sincospi.f32(float [[IN_VAL:%.*]])
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { float, float } [[CALL]], 0
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { float, float } [[CALL]], 1
; CHECK: store float [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 4
; CHECK: store float [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 4
; CHECK: [[EXIT:.*:]]
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds float, ptr %in, i64 %iv
%in_val = load float, ptr %arrayidx, align 4
%call = tail call { float, float } @llvm.sincospi.f32(float %in_val)
%extract_a = extractvalue { float, float } %call, 0
%extract_b = extractvalue { float, float } %call, 1
%arrayidx2 = getelementptr inbounds float, ptr %out_a, i64 %iv
store float %extract_a, ptr %arrayidx2, align 4
%arrayidx4 = getelementptr inbounds float, ptr %out_b, i64 %iv
store float %extract_b, ptr %arrayidx4, align 4
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %exit, label %for.body
exit:
ret void
}
define void @sincospi_f64(ptr noalias %in, ptr noalias writeonly %out_a, ptr noalias writeonly %out_b) {
; CHECK-LABEL: define void @sincospi_f64(
; CHECK-SAME: ptr noalias [[IN:%.*]], ptr noalias writeonly [[OUT_A:%.*]], ptr noalias writeonly [[OUT_B:%.*]]) {
; CHECK: [[ENTRY:.*:]]
; CHECK: [[VECTOR_PH:.*:]]
; CHECK: [[VECTOR_BODY:.*:]]
; CHECK: [[TMP3:%.*]] = call { <2 x double>, <2 x double> } @llvm.sincospi.v2f64(<2 x double> [[WIDE_LOAD:%.*]])
; CHECK: [[TMP4:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 0
; CHECK: [[TMP5:%.*]] = extractvalue { <2 x double>, <2 x double> } [[TMP3]], 1
; CHECK: store <2 x double> [[TMP4]], ptr [[TMP7:%.*]], align 8
; CHECK: store <2 x double> [[TMP5]], ptr [[TMP9:%.*]], align 8
; CHECK: [[MIDDLE_BLOCK:.*:]]
; CHECK: [[SCALAR_PH:.*:]]
; CHECK: [[FOR_BODY:.*:]]
; CHECK: [[CALL:%.*]] = tail call { double, double } @llvm.sincospi.f64(double [[IN_VAL:%.*]])
; CHECK: [[EXTRACT_A:%.*]] = extractvalue { double, double } [[CALL]], 0
; CHECK: [[EXTRACT_B:%.*]] = extractvalue { double, double } [[CALL]], 1
; CHECK: store double [[EXTRACT_A]], ptr [[ARRAYIDX2:%.*]], align 8
; CHECK: store double [[EXTRACT_B]], ptr [[ARRAYIDX4:%.*]], align 8
; CHECK: [[EXIT:.*:]]
;
entry:
br label %for.body
for.body:
%iv = phi i64 [ 0, %entry ], [ %iv.next, %for.body ]
%arrayidx = getelementptr inbounds double, ptr %in, i64 %iv
%in_val = load double, ptr %arrayidx, align 8
%call = tail call { double, double } @llvm.sincospi.f64(double %in_val)
%extract_a = extractvalue { double, double } %call, 0
%extract_b = extractvalue { double, double } %call, 1
%arrayidx2 = getelementptr inbounds double, ptr %out_a, i64 %iv
store double %extract_a, ptr %arrayidx2, align 8
%arrayidx4 = getelementptr inbounds double, ptr %out_b, i64 %iv
store double %extract_b, ptr %arrayidx4, align 8
%iv.next = add nuw nsw i64 %iv, 1
%exitcond.not = icmp eq i64 %iv.next, 1024
br i1 %exitcond.not, label %exit, label %for.body
exit:
ret void
}
Reference in New Issue View Git Blame Copy Permalink