shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/llvm/test/CodeGen/AArch64/neon-extmul.ll
David Green 3313cf4a83
[AArch64][GlobalISel] Add push_mul_through_s/zext (#141551) 
This extends the existing push_add_through_zext to handle mul, similar
to performVectorExtCombine in SDAG. This allows muls to be pushed up the
tree of extends, operating on smaller vector types whilst keeping the
result the same (providing there are > 2x bits in the output).

matchExtAddvToUdotAddv needs to be adjusted to make sure it keeps
generating dot instructions from add(ext(mul(ext, ext))).
2025-07-31 07:38:11 +01:00

344 lines
12 KiB
LLVM
Raw Permalink Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
; RUN: llc < %s -mtriple aarch64 -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
; RUN: llc < %s -mtriple aarch64 -o - -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
define <8 x i32> @extmuls_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: extmuls_v8i8_i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: sshll2 v1.4s, v0.8h, #0
; CHECK-SD-NEXT: sshll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmuls_v8i8_i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: smull v1.8h, v0.8b, v1.8b
; CHECK-GI-NEXT: sshll v0.4s, v1.4h, #0
; CHECK-GI-NEXT: sshll2 v1.4s, v1.8h, #0
; CHECK-GI-NEXT: ret
entry:
%s0s = sext <8 x i8> %s0 to <8 x i32>
%s1s = sext <8 x i8> %s1 to <8 x i32>
%m = mul <8 x i32> %s0s, %s1s
ret <8 x i32> %m
}
define <8 x i32> @extmulu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: extmulu_v8i8_i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: ushll2 v1.4s, v0.8h, #0
; CHECK-SD-NEXT: ushll v0.4s, v0.4h, #0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmulu_v8i8_i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: umull v1.8h, v0.8b, v1.8b
; CHECK-GI-NEXT: ushll v0.4s, v1.4h, #0
; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
; CHECK-GI-NEXT: ret
entry:
%s0s = zext <8 x i8> %s0 to <8 x i32>
%s1s = zext <8 x i8> %s1 to <8 x i32>
%m = mul <8 x i32> %s0s, %s1s
ret <8 x i32> %m
}
define <8 x i32> @extmulsu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: extmulsu_v8i8_i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: ushll v2.8h, v1.8b, #0
; CHECK-SD-NEXT: smull2 v1.4s, v0.8h, v2.8h
; CHECK-SD-NEXT: smull v0.4s, v0.4h, v2.4h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmulsu_v8i8_i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: sshll v2.8h, v0.8b, #0
; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT: smull v0.4s, v2.4h, v1.4h
; CHECK-GI-NEXT: smull2 v1.4s, v2.8h, v1.8h
; CHECK-GI-NEXT: ret
entry:
%s0s = sext <8 x i8> %s0 to <8 x i32>
%s1s = zext <8 x i8> %s1 to <8 x i32>
%m = mul <8 x i32> %s0s, %s1s
ret <8 x i32> %m
}
define <8 x i32> @extmuladds_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) {
; CHECK-SD-LABEL: extmuladds_v8i8_i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: saddw2 v1.4s, v3.4s, v0.8h
; CHECK-SD-NEXT: saddw v0.4s, v2.4s, v0.4h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmuladds_v8i8_i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: smull v1.8h, v0.8b, v1.8b
; CHECK-GI-NEXT: saddw v0.4s, v2.4s, v1.4h
; CHECK-GI-NEXT: saddw2 v1.4s, v3.4s, v1.8h
; CHECK-GI-NEXT: ret
entry:
%s0s = sext <8 x i8> %s0 to <8 x i32>
%s1s = sext <8 x i8> %s1 to <8 x i32>
%m = mul <8 x i32> %s0s, %s1s
%a = add <8 x i32> %m, %b
ret <8 x i32> %a
}
define <8 x i32> @extmuladdu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) {
; CHECK-SD-LABEL: extmuladdu_v8i8_i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: uaddw2 v1.4s, v3.4s, v0.8h
; CHECK-SD-NEXT: uaddw v0.4s, v2.4s, v0.4h
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmuladdu_v8i8_i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: umull v1.8h, v0.8b, v1.8b
; CHECK-GI-NEXT: uaddw v0.4s, v2.4s, v1.4h
; CHECK-GI-NEXT: uaddw2 v1.4s, v3.4s, v1.8h
; CHECK-GI-NEXT: ret
entry:
%s0s = zext <8 x i8> %s0 to <8 x i32>
%s1s = zext <8 x i8> %s1 to <8 x i32>
%m = mul <8 x i32> %s0s, %s1s
%a = add <8 x i32> %m, %b
ret <8 x i32> %a
}
define <8 x i32> @extmuladdsu_v8i8_i32(<8 x i8> %s0, <8 x i8> %s1, <8 x i32> %b) {
; CHECK-SD-LABEL: extmuladdsu_v8i8_i32:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: smlal2 v3.4s, v0.8h, v1.8h
; CHECK-SD-NEXT: smlal v2.4s, v0.4h, v1.4h
; CHECK-SD-NEXT: mov v0.16b, v2.16b
; CHECK-SD-NEXT: mov v1.16b, v3.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmuladdsu_v8i8_i32:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT: smlal v2.4s, v0.4h, v1.4h
; CHECK-GI-NEXT: smlal2 v3.4s, v0.8h, v1.8h
; CHECK-GI-NEXT: mov v0.16b, v2.16b
; CHECK-GI-NEXT: mov v1.16b, v3.16b
; CHECK-GI-NEXT: ret
entry:
%s0s = sext <8 x i8> %s0 to <8 x i32>
%s1s = zext <8 x i8> %s1 to <8 x i32>
%m = mul <8 x i32> %s0s, %s1s
%a = add <8 x i32> %m, %b
ret <8 x i32> %a
}
define <8 x i64> @extmuls_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: extmuls_v8i8_i64:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: sshll v1.4s, v0.4h, #0
; CHECK-SD-NEXT: sshll2 v2.4s, v0.8h, #0
; CHECK-SD-NEXT: sshll v0.2d, v1.2s, #0
; CHECK-SD-NEXT: sshll2 v3.2d, v2.4s, #0
; CHECK-SD-NEXT: sshll2 v1.2d, v1.4s, #0
; CHECK-SD-NEXT: sshll v2.2d, v2.2s, #0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmuls_v8i8_i64:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: smull v0.8h, v0.8b, v1.8b
; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0
; CHECK-GI-NEXT: sshll2 v3.4s, v0.8h, #0
; CHECK-GI-NEXT: sshll v0.2d, v1.2s, #0
; CHECK-GI-NEXT: sshll2 v1.2d, v1.4s, #0
; CHECK-GI-NEXT: sshll v2.2d, v3.2s, #0
; CHECK-GI-NEXT: sshll2 v3.2d, v3.4s, #0
; CHECK-GI-NEXT: ret
entry:
%s0s = sext <8 x i8> %s0 to <8 x i64>
%s1s = sext <8 x i8> %s1 to <8 x i64>
%m = mul <8 x i64> %s0s, %s1s
ret <8 x i64> %m
}
define <8 x i64> @extmulu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: extmulu_v8i8_i64:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: ushll v1.4s, v0.4h, #0
; CHECK-SD-NEXT: ushll2 v2.4s, v0.8h, #0
; CHECK-SD-NEXT: ushll v0.2d, v1.2s, #0
; CHECK-SD-NEXT: ushll2 v3.2d, v2.4s, #0
; CHECK-SD-NEXT: ushll2 v1.2d, v1.4s, #0
; CHECK-SD-NEXT: ushll v2.2d, v2.2s, #0
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmulu_v8i8_i64:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b
; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0
; CHECK-GI-NEXT: ushll2 v3.4s, v0.8h, #0
; CHECK-GI-NEXT: ushll v0.2d, v1.2s, #0
; CHECK-GI-NEXT: ushll2 v1.2d, v1.4s, #0
; CHECK-GI-NEXT: ushll v2.2d, v3.2s, #0
; CHECK-GI-NEXT: ushll2 v3.2d, v3.4s, #0
; CHECK-GI-NEXT: ret
entry:
%s0s = zext <8 x i8> %s0 to <8 x i64>
%s1s = zext <8 x i8> %s1 to <8 x i64>
%m = mul <8 x i64> %s0s, %s1s
ret <8 x i64> %m
}
define <8 x i64> @extaddsu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1) {
; CHECK-SD-LABEL: extaddsu_v8i8_i64:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-SD-NEXT: sshll v2.4s, v0.4h, #0
; CHECK-SD-NEXT: ushll v4.4s, v1.4h, #0
; CHECK-SD-NEXT: sshll2 v5.4s, v0.8h, #0
; CHECK-SD-NEXT: ushll2 v6.4s, v1.8h, #0
; CHECK-SD-NEXT: smull v0.2d, v2.2s, v4.2s
; CHECK-SD-NEXT: smull2 v1.2d, v2.4s, v4.4s
; CHECK-SD-NEXT: smull2 v3.2d, v5.4s, v6.4s
; CHECK-SD-NEXT: smull v2.2d, v5.2s, v6.2s
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extaddsu_v8i8_i64:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT: sshll v2.4s, v0.4h, #0
; CHECK-GI-NEXT: ushll v3.4s, v1.4h, #0
; CHECK-GI-NEXT: sshll2 v4.4s, v0.8h, #0
; CHECK-GI-NEXT: ushll2 v5.4s, v1.8h, #0
; CHECK-GI-NEXT: smull v0.2d, v2.2s, v3.2s
; CHECK-GI-NEXT: smull2 v1.2d, v2.4s, v3.4s
; CHECK-GI-NEXT: smull v2.2d, v4.2s, v5.2s
; CHECK-GI-NEXT: smull2 v3.2d, v4.4s, v5.4s
; CHECK-GI-NEXT: ret
entry:
%s0s = sext <8 x i8> %s0 to <8 x i64>
%s1s = zext <8 x i8> %s1 to <8 x i64>
%m = mul <8 x i64> %s0s, %s1s
ret <8 x i64> %m
}
define <8 x i64> @extmuladds_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) {
; CHECK-SD-LABEL: extmuladds_v8i8_i64:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: smull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: sshll2 v6.4s, v0.8h, #0
; CHECK-SD-NEXT: sshll v1.4s, v0.4h, #0
; CHECK-SD-NEXT: saddw2 v5.2d, v5.2d, v6.4s
; CHECK-SD-NEXT: saddw v0.2d, v2.2d, v1.2s
; CHECK-SD-NEXT: saddw2 v1.2d, v3.2d, v1.4s
; CHECK-SD-NEXT: saddw v2.2d, v4.2d, v6.2s
; CHECK-SD-NEXT: mov v3.16b, v5.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmuladds_v8i8_i64:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: smull v0.8h, v0.8b, v1.8b
; CHECK-GI-NEXT: sshll v1.4s, v0.4h, #0
; CHECK-GI-NEXT: sshll2 v6.4s, v0.8h, #0
; CHECK-GI-NEXT: saddw v0.2d, v2.2d, v1.2s
; CHECK-GI-NEXT: saddw2 v1.2d, v3.2d, v1.4s
; CHECK-GI-NEXT: saddw v2.2d, v4.2d, v6.2s
; CHECK-GI-NEXT: saddw2 v3.2d, v5.2d, v6.4s
; CHECK-GI-NEXT: ret
entry:
%s0s = sext <8 x i8> %s0 to <8 x i64>
%s1s = sext <8 x i8> %s1 to <8 x i64>
%m = mul <8 x i64> %s0s, %s1s
%a = add <8 x i64> %m, %b
ret <8 x i64> %a
}
define <8 x i64> @extmuladdu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) {
; CHECK-SD-LABEL: extmuladdu_v8i8_i64:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: umull v0.8h, v0.8b, v1.8b
; CHECK-SD-NEXT: ushll2 v6.4s, v0.8h, #0
; CHECK-SD-NEXT: ushll v1.4s, v0.4h, #0
; CHECK-SD-NEXT: uaddw2 v5.2d, v5.2d, v6.4s
; CHECK-SD-NEXT: uaddw v0.2d, v2.2d, v1.2s
; CHECK-SD-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
; CHECK-SD-NEXT: uaddw v2.2d, v4.2d, v6.2s
; CHECK-SD-NEXT: mov v3.16b, v5.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmuladdu_v8i8_i64:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: umull v0.8h, v0.8b, v1.8b
; CHECK-GI-NEXT: ushll v1.4s, v0.4h, #0
; CHECK-GI-NEXT: ushll2 v6.4s, v0.8h, #0
; CHECK-GI-NEXT: uaddw v0.2d, v2.2d, v1.2s
; CHECK-GI-NEXT: uaddw2 v1.2d, v3.2d, v1.4s
; CHECK-GI-NEXT: uaddw v2.2d, v4.2d, v6.2s
; CHECK-GI-NEXT: uaddw2 v3.2d, v5.2d, v6.4s
; CHECK-GI-NEXT: ret
entry:
%s0s = zext <8 x i8> %s0 to <8 x i64>
%s1s = zext <8 x i8> %s1 to <8 x i64>
%m = mul <8 x i64> %s0s, %s1s
%a = add <8 x i64> %m, %b
ret <8 x i64> %a
}
define <8 x i64> @extmuladdsu_v8i8_i64(<8 x i8> %s0, <8 x i8> %s1, <8 x i64> %b) {
; CHECK-SD-LABEL: extmuladdsu_v8i8_i64:
; CHECK-SD: // %bb.0: // %entry
; CHECK-SD-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-SD-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-SD-NEXT: sshll v6.4s, v0.4h, #0
; CHECK-SD-NEXT: ushll v7.4s, v1.4h, #0
; CHECK-SD-NEXT: sshll2 v0.4s, v0.8h, #0
; CHECK-SD-NEXT: ushll2 v1.4s, v1.8h, #0
; CHECK-SD-NEXT: smlal v2.2d, v6.2s, v7.2s
; CHECK-SD-NEXT: smlal2 v3.2d, v6.4s, v7.4s
; CHECK-SD-NEXT: smlal2 v5.2d, v0.4s, v1.4s
; CHECK-SD-NEXT: smlal v4.2d, v0.2s, v1.2s
; CHECK-SD-NEXT: mov v0.16b, v2.16b
; CHECK-SD-NEXT: mov v1.16b, v3.16b
; CHECK-SD-NEXT: mov v2.16b, v4.16b
; CHECK-SD-NEXT: mov v3.16b, v5.16b
; CHECK-SD-NEXT: ret
;
; CHECK-GI-LABEL: extmuladdsu_v8i8_i64:
; CHECK-GI: // %bb.0: // %entry
; CHECK-GI-NEXT: sshll v0.8h, v0.8b, #0
; CHECK-GI-NEXT: ushll v1.8h, v1.8b, #0
; CHECK-GI-NEXT: sshll v6.4s, v0.4h, #0
; CHECK-GI-NEXT: ushll v7.4s, v1.4h, #0
; CHECK-GI-NEXT: sshll2 v0.4s, v0.8h, #0
; CHECK-GI-NEXT: ushll2 v1.4s, v1.8h, #0
; CHECK-GI-NEXT: smlal v2.2d, v6.2s, v7.2s
; CHECK-GI-NEXT: smlal2 v3.2d, v6.4s, v7.4s
; CHECK-GI-NEXT: smlal v4.2d, v0.2s, v1.2s
; CHECK-GI-NEXT: smlal2 v5.2d, v0.4s, v1.4s
; CHECK-GI-NEXT: mov v0.16b, v2.16b
; CHECK-GI-NEXT: mov v1.16b, v3.16b
; CHECK-GI-NEXT: mov v2.16b, v4.16b
; CHECK-GI-NEXT: mov v3.16b, v5.16b
; CHECK-GI-NEXT: ret
entry:
%s0s = sext <8 x i8> %s0 to <8 x i64>
%s1s = zext <8 x i8> %s1 to <8 x i64>
%m = mul <8 x i64> %s0s, %s1s
%a = add <8 x i64> %m, %b
ret <8 x i64> %a
}
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; CHECK: {{.*}}
Reference in New Issue View Git Blame Copy Permalink