shylie/llvm-project
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
llvm-project/llvm/test/CodeGen/AMDGPU/vector-reduce-mul.ll
Brox Chen 6dbc01e801
[AMDGPU][True16][CodeGen] update GFX11Plus codegen test with true16 flag (#135078) 
This is a NFC patch.

This patch run a bulk update on CodeGen tests that are impacted by the
true16 features. This patch applies:
1. duplicate GFX11plus runlines and apply them with
"+mattr=+real-true16" and "+mattr=-real-true16"
2. update the test with the update script

For some GISEL runlines, the current CodeGen do not fully support the
true16 version. Still update the runlines, but comment out the failing
one, and added a "FIXME-TRUE16" comment to that test for easier
tracking. These test will be fixed in the following patches.

This is in a transition state that we support both
"+real-true16/-real-true16" in our code base. We plan to move to
"+real-true16" as default, and finally remove "-real-true16" mode and
test lines.
2025-04-23 13:06:52 -04:00

4340 lines
213 KiB
LLVM
Raw Permalink Blame History

; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx700 < %s | FileCheck -check-prefixes=GFX7,GFX7-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx801 < %s | FileCheck -check-prefixes=GFX8,GFX8-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx942 < %s | FileCheck -check-prefixes=GFX9,GFX9-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-SDAG %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1010 < %s | FileCheck -check-prefixes=GFX10,GFX10-GISEL %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-SDAG,GFX11-SDAG-FAKE16 %s
; FIXME-TRUE16. enable gisel
; XUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-TRUE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1100 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX11,GFX11-GISEL,GFX11-GISEL-FAKE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-TRUE16 %s
; RUN: llc -global-isel=0 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-SDAG,GFX12-SDAG-FAKE16 %s
; XUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=+real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-TRUE16 %s
; RUN: llc -global-isel=1 -mtriple=amdgcn -mcpu=gfx1200 -mattr=-real-true16 < %s | FileCheck -check-prefixes=GFX12,GFX12-GISEL,GFX12-GISEL-FAKE16 %s
define i8 @test_vector_reduce_mul_v2i8(<2 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v2i8:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v2i8:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v2i8:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v1, v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX8-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v2i8:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v2i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-SDAG-NEXT: v_mul_lo_u16_sdwa v1, v1, v1 dst_sel:BYTE_1 dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:BYTE_1
; GFX9-SDAG-NEXT: s_nop 0
; GFX9-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v2i8:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v2i8:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_lshrrev_b16 v2, 8, v1
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: v_mul_lo_u16 v2, v1, v2
; GFX10-SDAG-NEXT: v_lshlrev_b16 v1, 8, v2
; GFX10-SDAG-NEXT: v_or_b32_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:BYTE_0 src1_sel:DWORD
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v2i8:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v2i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v2i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b16 v2, 8, v1
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v2
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX11-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v2i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v2i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b16 v0.h, 8, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshlrev_b16 v0.h, 8, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_or_b16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v2i8:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b16 v2, 8, v1
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v2
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_lshlrev_b16 v1, 8, v1
; GFX12-SDAG-FAKE16-NEXT: v_or_b32_e32 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v2i8:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v2i8(<2 x i8> %v)
ret i8 %res
}
define i8 @test_vector_reduce_mul_v3i8(<3 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v3i8:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v1, v0
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v3i8:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v3i8:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v1, v0
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v3i8:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v3i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v1, v0
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v3i8:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v3i8:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v1, v0
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v3i8:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v3i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v1, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v3i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v1.l, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v3i8:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v1, v0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i8:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v3i8(<3 x i8> %v)
ret i8 %res
}
define i8 @test_vector_reduce_mul_v4i8(<4 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v4i8:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v4i8:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v4i8:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v4i8:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v4i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v4i8:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v4i8:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v4i8:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v4i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v4i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v4i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v4i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v1.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v4i8:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i8:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v4i8(<4 x i8> %v)
ret i8 %res
}
define i8 @test_vector_reduce_mul_v8i8(<8 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v8i8:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v8i8:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v8i8:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v8i8:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v8i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v8i8:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v8i8:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX10-SDAG-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX10-SDAG-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX10-SDAG-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v8i8:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX10-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX10-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX10-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v8i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v2.l, v6.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v7.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v5.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v4.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v8i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v8i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v8i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v2.l, v6.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v3.l, v7.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v5.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v4.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v8i8:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i8:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v8i8(<8 x i8> %v)
ret i8 %res
}
define i8 @test_vector_reduce_mul_v16i8(<16 x i8> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v16i8:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX7-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX7-SDAG-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX7-SDAG-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v16i8:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX7-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX7-GISEL-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX7-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX7-GISEL-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v16i8:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v5, v13
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v9
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v7, v7, v15
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v11
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v4, v12
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v8
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v6, v6, v14
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v10
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v16i8:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v8
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v9
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v2, v10
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v3, v3, v11
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v4, v12
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v5, v5, v13
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v6, v14
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v7, v7, v15
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v16i8:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v5, v5, v13
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v9
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v7, v7, v15
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v11
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v4, v4, v12
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v8
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v6, v6, v14
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v10
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX9-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v16i8:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v8
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v9
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v2, v2, v10
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v3, v3, v11
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v4, v4, v12
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v5, v5, v13
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v6, v6, v14
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v7, v7, v15
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v16i8:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u16 v5, v5, v13
; GFX10-SDAG-NEXT: v_mul_lo_u16 v1, v1, v9
; GFX10-SDAG-NEXT: v_mul_lo_u16 v7, v7, v15
; GFX10-SDAG-NEXT: v_mul_lo_u16 v6, v6, v14
; GFX10-SDAG-NEXT: v_mul_lo_u16 v2, v2, v10
; GFX10-SDAG-NEXT: v_mul_lo_u16 v3, v3, v11
; GFX10-SDAG-NEXT: v_mul_lo_u16 v4, v4, v12
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v8
; GFX10-SDAG-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX10-SDAG-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX10-SDAG-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX10-SDAG-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v16i8:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v8
; GFX10-GISEL-NEXT: v_mul_lo_u16 v1, v1, v9
; GFX10-GISEL-NEXT: v_mul_lo_u16 v2, v2, v10
; GFX10-GISEL-NEXT: v_mul_lo_u16 v3, v3, v11
; GFX10-GISEL-NEXT: v_mul_lo_u16 v4, v4, v12
; GFX10-GISEL-NEXT: v_mul_lo_u16 v5, v5, v13
; GFX10-GISEL-NEXT: v_mul_lo_u16 v6, v6, v14
; GFX10-GISEL-NEXT: v_mul_lo_u16 v7, v7, v15
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX10-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX10-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX10-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i8:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v5.l, v13.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v9.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v5.l, v7.l, v15.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v6.l, v14.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v10.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v3.l, v11.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v4.l, v12.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v1.h
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v2.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.h, v5.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.l
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v2.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v16i8:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v5, v5, v13
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v9
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v7, v7, v15
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v6, v6, v14
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v10
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v11
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v4, v4, v12
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v8
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v16i8:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v8
; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v9
; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v10
; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v11
; GFX11-GISEL-NEXT: v_mul_lo_u16 v4, v4, v12
; GFX11-GISEL-NEXT: v_mul_lo_u16 v5, v5, v13
; GFX11-GISEL-NEXT: v_mul_lo_u16 v6, v6, v14
; GFX11-GISEL-NEXT: v_mul_lo_u16 v7, v7, v15
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX11-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i8:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.h, v5.l, v13.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v9.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v5.l, v7.l, v15.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v6.l, v14.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.l, v10.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.h, v3.l, v11.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v3.l, v4.l, v12.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v8.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v1.h
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.h, v2.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v2.l, v2.h, v5.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v3.l
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v1.l, v1.l, v2.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v1.l
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_and_b16 v0.l, 0xff, v0.l
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v16i8:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v5, v5, v13
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v9
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v7, v7, v15
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v6, v6, v14
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v10
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v11
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v4, v4, v12
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v8
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_and_b32_e32 v0, 0xff, v0
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i8:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v8
; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v9
; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v10
; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v11
; GFX12-GISEL-NEXT: v_mul_lo_u16 v4, v4, v12
; GFX12-GISEL-NEXT: v_mul_lo_u16 v5, v5, v13
; GFX12-GISEL-NEXT: v_mul_lo_u16 v6, v6, v14
; GFX12-GISEL-NEXT: v_mul_lo_u16 v7, v7, v15
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v4
; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v5
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v2, v2, v6
; GFX12-GISEL-NEXT: v_mul_lo_u16 v3, v3, v7
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-GISEL-NEXT: v_mul_lo_u16 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 8
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i8 @llvm.vector.reduce.mul.v16i8(<16 x i8> %v)
ret i8 %res
}
define i16 @test_vector_reduce_mul_v2i16(<2 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v2i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v1
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v1, 16, v1
; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v2i16:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_vector_reduce_mul_v2i16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v2i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v2i16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v2i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_alignbit_b32 v1, s4, v0, 16
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v2i16:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v2i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v2i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v2i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v2i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v2i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v2i16:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v2i16(<2 x i16> %v)
ret i16 %res
}
define i16 @test_vector_reduce_mul_v3i16(<3 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v3i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_lshlrev_b32_e32 v3, 16, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-SDAG-NEXT: v_and_b32_e32 v2, 0xffff, v2
; GFX7-SDAG-NEXT: v_or_b32_e32 v0, v0, v3
; GFX7-SDAG-NEXT: v_or_b32_e32 v2, 0x10000, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_and_b32_e32 v1, 0xffff, v1
; GFX7-SDAG-NEXT: v_lshrrev_b32_e32 v2, 16, v2
; GFX7-SDAG-NEXT: v_mul_u32_u24_e32 v1, v1, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v3i16:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_vector_reduce_mul_v3i16:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX8-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v3i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mov_b32_e32 v2, 0x5040100
; GFX9-SDAG-NEXT: v_perm_b32 v1, 1, v1, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-SDAG-NEXT: s_nop 0
; GFX9-SDAG-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v3i16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v0 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:DWORD src1_sel:WORD_1
; GFX9-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v3i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_perm_b32 v1, 1, v1, 0x5040100
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX10-SDAG-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v3i16:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v3i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_perm_b32 v1, 1, v1, 0x5040100
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v3i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v3i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_mov_b16_e32 v1.h, 1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: v_mul_lo_u16 v0.l, v0.l, v0.h
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v3i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_perm_b32 v1, 1, v1, 0x5040100
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i16:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v2, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v2
; GFX12-GISEL-NEXT: v_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v3i16(<3 x i16> %v)
ret i16 %res
}
define i16 @test_vector_reduce_mul_v4i16(<4 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v4i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v4i16:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v4i16:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v2, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v4i16:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v0, v1
; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v1 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v4i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-SDAG-NEXT: s_nop 0
; GFX9-SDAG-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v4i16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-GISEL-NEXT: s_nop 0
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v4i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: v_alignbit_b32 v1, s4, v0, 16
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v4i16:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v4i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v4i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v4i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v4i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v4i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i16:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v4i16(<4 x i16> %v)
ret i16 %res
}
define i16 @test_vector_reduce_mul_v8i16(<8 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v8i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v8i16:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v8i16:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v4, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v5, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v5, v4
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v8i16:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v0, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v2 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v1, v3
; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v1, v1, v3 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v4, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v8i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-SDAG-NEXT: s_nop 0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-SDAG-NEXT: s_nop 0
; GFX9-SDAG-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v8i16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX9-GISEL-NEXT: s_nop 0
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-GISEL-NEXT: s_nop 0
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v8i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: v_alignbit_b32 v1, s4, v0, 16
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v8i16:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v8i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v8i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v8i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v8i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v8i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i16:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v8i16(<8 x i16> %v)
ret i16 %res
}
define i16 @test_vector_reduce_mul_v16i16(<16 x i16> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v16i16:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX7-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX7-SDAG-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX7-SDAG-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: v_and_b32_e32 v0, 0xffff, v0
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v16i16:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX7-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX7-GISEL-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX7-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX7-GISEL-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: v_bfe_u32 v0, v0, 0, 16
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v16i16:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v8, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v9, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v10, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-SDAG-NEXT: v_mul_lo_u16_sdwa v11, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v2, v6
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v4
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v3, v3, v7
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v5
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v4, v11, v10
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v5, v9, v8
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v2, v5, v4
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-SDAG-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v16i16:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v8, v0, v4
; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v0, v0, v4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v4, v1, v5
; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v1, v1, v5 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v5, v2, v6
; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v2, v2, v6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v6, v3, v7
; GFX8-GISEL-NEXT: v_mul_lo_u16_sdwa v3, v3, v7 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_1 src1_sel:WORD_1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v5, v8, v5
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v4, v6
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v1, v1, v3
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v2, v5, v2
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v0, v1
; GFX8-GISEL-NEXT: v_mul_lo_u16_e32 v0, v2, v0
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v16i16:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX9-SDAG-NEXT: s_nop 0
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-SDAG-NEXT: s_nop 0
; GFX9-SDAG-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX9-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v16i16:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX9-GISEL-NEXT: s_nop 0
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-GISEL-NEXT: s_nop 0
; GFX9-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX9-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v16i16:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: v_alignbit_b32 v1, s4, v0, 16
; GFX10-SDAG-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v16i16:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX10-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i16:
; GFX11-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX11-SDAG-TRUE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-TRUE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX11-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v16i16:
; GFX11-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX11-SDAG-FAKE16-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX11-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v16i16:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-TRUE16-LABEL: test_vector_reduce_mul_v16i16:
; GFX12-SDAG-TRUE16: ; %bb.0: ; %entry
; GFX12-SDAG-TRUE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-SDAG-TRUE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-TRUE16-NEXT: v_lshl_or_b32 v1, s0, 16, v1
; GFX12-SDAG-TRUE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-TRUE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-FAKE16-LABEL: test_vector_reduce_mul_v16i16:
; GFX12-SDAG-FAKE16: ; %bb.0: ; %entry
; GFX12-SDAG-FAKE16-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: v_alignbit_b32 v1, s0, v0, 16
; GFX12-SDAG-FAKE16-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-FAKE16-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-SDAG-FAKE16-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i16:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v4
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v5
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v2, v2, v6
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v3, v3, v7
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v2
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: v_lshrrev_b32_e32 v1, 16, v0
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_pk_mul_lo_u16 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i16 @llvm.vector.reduce.mul.v16i16(<16 x i16> %v)
ret i16 %res
}
define i32 @test_vector_reduce_mul_v2i32(<2 x i32> %v) {
; GFX7-LABEL: test_vector_reduce_mul_v2i32:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_vector_reduce_mul_v2i32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_vector_reduce_mul_v2i32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_mul_v2i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: test_vector_reduce_mul_v2i32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: test_vector_reduce_mul_v2i32:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i32 @llvm.vector.reduce.mul.v2i32(<2 x i32> %v)
ret i32 %res
}
define i32 @test_vector_reduce_mul_v3i32(<3 x i32> %v) {
; GFX7-LABEL: test_vector_reduce_mul_v3i32:
; GFX7: ; %bb.0: ; %entry
; GFX7-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-LABEL: test_vector_reduce_mul_v3i32:
; GFX8: ; %bb.0: ; %entry
; GFX8-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX8-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-LABEL: test_vector_reduce_mul_v3i32:
; GFX9: ; %bb.0: ; %entry
; GFX9-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX9-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-LABEL: test_vector_reduce_mul_v3i32:
; GFX10: ; %bb.0: ; %entry
; GFX10-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX10-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX10-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-LABEL: test_vector_reduce_mul_v3i32:
; GFX11: ; %bb.0: ; %entry
; GFX11-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX11-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX11-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-LABEL: test_vector_reduce_mul_v3i32:
; GFX12: ; %bb.0: ; %entry
; GFX12-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-NEXT: s_wait_expcnt 0x0
; GFX12-NEXT: s_wait_samplecnt 0x0
; GFX12-NEXT: s_wait_bvhcnt 0x0
; GFX12-NEXT: s_wait_kmcnt 0x0
; GFX12-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX12-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX12-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i32 @llvm.vector.reduce.mul.v3i32(<3 x i32> %v)
ret i32 %res
}
define i32 @test_vector_reduce_mul_v4i32(<4 x i32> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v4i32:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v4i32:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v4i32:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v4i32:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX8-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v4i32:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v4i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX9-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v4i32:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v4i32:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: test_vector_reduce_mul_v4i32:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v4i32:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-LABEL: test_vector_reduce_mul_v4i32:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i32:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i32 @llvm.vector.reduce.mul.v4i32(<4 x i32> %v)
ret i32 %res
}
define i32 @test_vector_reduce_mul_v8i32(<8 x i32> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v8i32:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v8i32:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v8i32:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v8i32:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX8-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX8-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX8-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX8-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v8i32:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX9-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX9-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v8i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX9-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX9-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX9-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX9-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v8i32:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX10-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX10-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v8i32:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: test_vector_reduce_mul_v8i32:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v8i32:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-LABEL: test_vector_reduce_mul_v8i32:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX12-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i32:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i32 @llvm.vector.reduce.mul.v8i32(<8 x i32> %v)
ret i32 %res
}
define i32 @test_vector_reduce_mul_v16i32(<16 x i32> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v16i32:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX7-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX7-SDAG-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX7-SDAG-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v16i32:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX7-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX7-GISEL-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX7-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX7-GISEL-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX7-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX7-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX7-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX7-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v16i32:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX8-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX8-SDAG-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX8-SDAG-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v16i32:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX8-GISEL-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX8-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX8-GISEL-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX8-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX8-GISEL-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX8-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX8-GISEL-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX8-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX8-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX8-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX8-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX8-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v16i32:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX9-SDAG-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX9-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX9-SDAG-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX9-SDAG-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX9-SDAG-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX9-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX9-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX9-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v16i32:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX9-GISEL-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX9-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX9-GISEL-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX9-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX9-GISEL-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX9-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX9-GISEL-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX9-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX9-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX9-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX9-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX9-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v16i32:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX10-SDAG-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX10-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX10-SDAG-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX10-SDAG-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX10-SDAG-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX10-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX10-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX10-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX10-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v16i32:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX10-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX10-GISEL-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX10-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX10-GISEL-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX10-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX10-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX10-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX10-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: test_vector_reduce_mul_v16i32:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v16i32:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX11-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX11-GISEL-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX11-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX11-GISEL-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX11-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX11-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-LABEL: test_vector_reduce_mul_v16i32:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX12-SDAG-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX12-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX12-SDAG-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX12-SDAG-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX12-SDAG-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX12-SDAG-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX12-SDAG-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i32:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v8
; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v9
; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v2, v10
; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v3, v11
; GFX12-GISEL-NEXT: v_mul_lo_u32 v4, v4, v12
; GFX12-GISEL-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX12-GISEL-NEXT: v_mul_lo_u32 v6, v6, v14
; GFX12-GISEL-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v4
; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v5
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-GISEL-NEXT: v_mul_lo_u32 v2, v2, v6
; GFX12-GISEL-NEXT: v_mul_lo_u32 v3, v3, v7
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX12-GISEL-NEXT: v_mul_lo_u32 v1, v1, v3
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v1
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i32 @llvm.vector.reduce.mul.v16i32(<16 x i32> %v)
ret i32 %res
}
define i64 @test_vector_reduce_mul_v2i64(<2 x i64> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v2i64:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mov_b32_e32 v4, v1
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v4, v2
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v3
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v2i64:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, v0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v5, v1
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v3, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v2, v[3:4]
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v2i64:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mov_b32_e32 v4, v1
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v2, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v4, v2
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v3
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v2i64:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, v0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, v1
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v3, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v2, v[3:4]
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v2i64:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v2, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v2i64:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mov_b32_e32 v4, v0
; GFX9-GISEL-NEXT: v_mov_b32_e32 v5, v1
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v4, v3, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v4, v2, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v5, v2, v[6:7]
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v2i64:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v2, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v2i64:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v5, v1
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], s4, v4, v2, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], s4, v4, v3, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v5, v2, v[3:4]
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: test_vector_reduce_mul_v2i64:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_mov_b32_e32 v4, v0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v1, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v4, v3
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0
; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v3, v5
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v2i64:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_dual_mov_b32 v4, v0 :: v_dual_mov_b32 v5, v2
; GFX11-GISEL-NEXT: v_mov_b32_e32 v6, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v4, v5, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v4, v3, v[1:2]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v6, v5, v[7:8]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-LABEL: test_vector_reduce_mul_v2i64:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v2i64:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_hi_u32 v4, v0, v2
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[3:4], null, v0, v3, v[4:5]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v0, v2
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v1, v2, v[3:4]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v2i64(<2 x i64> %v)
ret i64 %res
}
define i64 @test_vector_reduce_mul_v3i64(<3 x i64> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v3i64:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v0, v2, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v1, v2
; GFX7-SDAG-NEXT: v_add_i32_e32 v3, vcc, v7, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v5, v6, v5
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v4, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v2, vcc, v3, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v4
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v5
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v3i64:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v0, v2, 0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v6, v1
; GFX7-GISEL-NEXT: v_mov_b32_e32 v1, v8
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v3, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v4, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v2, v[8:9]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v5, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v2, v4, v[5:6]
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v3i64:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v0, v2, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v1, v2
; GFX8-SDAG-NEXT: v_add_u32_e32 v3, vcc, v7, v3
; GFX8-SDAG-NEXT: v_mul_lo_u32 v5, v6, v5
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v4, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v2, vcc, v3, v2
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v2, v4
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v5
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v3i64:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v0, v2, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v6, v1
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, v8
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v3, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v4, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v2, v[8:9]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v5, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v2, v4, v[5:6]
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v3i64:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u32 v6, v1, v2
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v2, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v3, v6
; GFX9-SDAG-NEXT: v_mul_lo_u32 v2, v1, v4
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v0, v5
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v4, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v3, v2
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v3i64:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v0, v3, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v0, v2, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v1, v2, v[8:9]
; GFX9-GISEL-NEXT: v_add_u32_e32 v7, v7, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v6, v5, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v6, v4, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v7, v4, v[2:3]
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v3i64:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u32 v6, v1, v2
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v2, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v3, v6
; GFX10-SDAG-NEXT: v_mul_lo_u32 v2, v0, v5
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v1, v4
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v4, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v2, v3
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v3i64:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mov_b32_e32 v6, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v7, v1
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[8:9], s4, v6, v2, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[9:10], s4, v6, v3, v[9:10]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], s4, v8, v4, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], s4, v7, v2, v[9:10]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[5:6], s4, v8, v5, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v2, v4, v[5:6]
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: test_vector_reduce_mul_v3i64:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[6:7], null, v0, v2, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_add3_u32 v0, v7, v3, v1
; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v6, v5
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v0, v4
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v6, v4, 0
; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v2, v3
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v3i64:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_dual_mov_b32 v6, v0 :: v_dual_mov_b32 v7, v1
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v6, v2, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v8, v4, 0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[10:11], null, v6, v3, v[9:10]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[11:12], null, v7, v2, v[10:11]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v8, v5, v[1:2]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v11, v4, v[6:7]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-LABEL: test_vector_reduce_mul_v3i64:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v1, v2
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v3, v6
; GFX12-SDAG-NEXT: v_mul_lo_u32 v2, v0, v5
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v1, v4
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v4, 0
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v2, v3
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v3i64:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_hi_u32 v6, v0, v2
; GFX12-GISEL-NEXT: v_mul_lo_u32 v8, v0, v2
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[6:7], null, v0, v3, v[6:7]
; GFX12-GISEL-NEXT: v_mul_hi_u32 v0, v8, v4
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v1, v2, v[6:7]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[2:3], null, v8, v5, v[0:1]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v8, v4
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v1, v4, v[2:3]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v3i64(<3 x i64> %v)
ret i64 %res
}
define i64 @test_vector_reduce_mul_v4i64(<4 x i64> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v4i64:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v5, v0, v5
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v4, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v2, v7
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, v9, v5
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v2, v6, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v1, v4
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v3, v6
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v10, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v2, vcc, v5, v2
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v8, v0
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v9, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v9
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v3
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v4i64:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v0, v4, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v2, v6, 0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v8, v1
; GFX7-GISEL-NEXT: v_mov_b32_e32 v1, v10
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v0, v5, v[1:2]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v0, v12
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v2, v7, v[0:1]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v11, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v6, v[15:16]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v4, v[13:14]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v9, v2, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v3, v11, v[1:2]
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v4i64:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u32 v5, v0, v5
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v4, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v2, v7
; GFX8-SDAG-NEXT: v_add_u32_e32 v5, vcc, v9, v5
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v2, v6, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v1, v4
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v3, v6
; GFX8-SDAG-NEXT: v_add_u32_e32 v0, vcc, v10, v0
; GFX8-SDAG-NEXT: v_add_u32_e32 v2, vcc, v5, v2
; GFX8-SDAG-NEXT: v_add_u32_e32 v0, vcc, v0, v1
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v8, v0
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v9, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v2, v9
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v3
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v4i64:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v0, v4, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v2, v6, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v8, v1
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, v10
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v0, v5, v[1:2]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, v12
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[15:16], s[4:5], v2, v7, v[0:1]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v9, v11, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v6, v[15:16]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v4, v[13:14]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v9, v2, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v3, v11, v[1:2]
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v4i64:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u32 v8, v3, v6
; GFX9-SDAG-NEXT: v_mul_lo_u32 v7, v2, v7
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v2, v6, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v6, v1, v4
; GFX9-SDAG-NEXT: v_mul_lo_u32 v5, v0, v5
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v4, 0
; GFX9-SDAG-NEXT: v_add3_u32 v3, v3, v7, v8
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v5, v6
; GFX9-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v2, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v4i64:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[10:11], s[0:1], v0, v5, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v0, v4, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v1, v4, v[10:11]
; GFX9-GISEL-NEXT: v_add_u32_e32 v9, v9, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v7, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v2, v6, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v3, v6, v[0:1]
; GFX9-GISEL-NEXT: v_add_u32_e32 v2, v5, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v8, v2, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v8, v4, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v9, v4, v[2:3]
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v4i64:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u32 v8, v3, v6
; GFX10-SDAG-NEXT: v_mul_lo_u32 v7, v2, v7
; GFX10-SDAG-NEXT: v_mul_lo_u32 v9, v1, v4
; GFX10-SDAG-NEXT: v_mul_lo_u32 v5, v0, v5
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v4, 0
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[2:3], s4, v2, v6, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v5, v9
; GFX10-SDAG-NEXT: v_add3_u32 v3, v3, v7, v8
; GFX10-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v2, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v4i64:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[10:11], s4, v2, v6, 0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v8, v0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v9, v1
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[12:13], s4, v8, v4, 0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v11
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[14:15], s4, v2, v7, v[0:1]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v13
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], s4, v12, v10, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[7:8], s4, v8, v5, v[2:3]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], s4, v3, v6, v[14:15]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], s4, v9, v4, v[7:8]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v12, v2, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v3, v10, v[1:2]
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: test_vector_reduce_mul_v4i64:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v9, v3, v6
; GFX11-SDAG-NEXT: v_mul_lo_u32 v10, v2, v7
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v4
; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v0, v5
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[7:8], null, v0, v4, 0
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v2, v6, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_add3_u32 v0, v8, v5, v1
; GFX11-SDAG-NEXT: v_add3_u32 v1, v4, v10, v9
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v3
; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v7, v1
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v7, v3, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v4, v2
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v4i64:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_dual_mov_b32 v8, v0 :: v_dual_mov_b32 v9, v1
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[10:11], null, v2, v6, 0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[12:13], null, v8, v4, 0
; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v11
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[14:15], null, v2, v7, v[0:1]
; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, v13
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v12, v10, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[15:16], null, v8, v5, v[2:3]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v3, v6, v[14:15]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v9, v4, v[15:16]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v12, v7, v[1:2]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v5, v10, v[3:4]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-LABEL: test_vector_reduce_mul_v4i64:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v8, v3, v6
; GFX12-SDAG-NEXT: v_mul_lo_u32 v7, v2, v7
; GFX12-SDAG-NEXT: v_mul_lo_u32 v9, v1, v4
; GFX12-SDAG-NEXT: v_mul_lo_u32 v5, v0, v5
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v4, 0
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v2, v6, 0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v5, v9
; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v7, v8
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v4i64:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_hi_u32 v8, v2, v6
; GFX12-GISEL-NEXT: v_mul_hi_u32 v9, v0, v4
; GFX12-GISEL-NEXT: v_mul_lo_u32 v10, v0, v4
; GFX12-GISEL-NEXT: v_mul_lo_u32 v11, v2, v6
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[7:8], null, v2, v7, v[8:9]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[8:9], null, v0, v5, v[9:10]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mul_hi_u32 v0, v10, v11
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[2:3], null, v3, v6, v[7:8]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[3:4], null, v1, v4, v[8:9]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v10, v2, v[0:1]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v10, v11
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v3, v11, v[1:2]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v4i64(<4 x i64> %v)
ret i64 %res
}
define i64 @test_vector_reduce_mul_v8i64(<8 x i64> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v8i64:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v11, v2, v11
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v2, v10, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v10, v3, v10
; GFX7-SDAG-NEXT: v_mul_lo_u32 v15, v6, v15
; GFX7-SDAG-NEXT: v_add_i32_e32 v11, vcc, v17, v11
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v14, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v10, vcc, v11, v10
; GFX7-SDAG-NEXT: v_mul_lo_u32 v11, v7, v14
; GFX7-SDAG-NEXT: v_add_i32_e32 v3, vcc, v3, v15
; GFX7-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v0, v8, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v11, vcc, v3, v11
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v4, v13
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v12, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v8, v1, v8
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v5, v12
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v4, v0
; GFX7-SDAG-NEXT: v_add_i32_e32 v7, vcc, v7, v9
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v0, v1
; GFX7-SDAG-NEXT: v_mul_lo_u32 v4, v6, v0
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v6, vcc, v7, v8
; GFX7-SDAG-NEXT: v_add_i32_e32 v7, vcc, v1, v4
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v16, v11
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v16, v2, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v10, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v6, v3
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v5, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v4, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v3, vcc, v7, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v3, v4
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v3
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v8i64:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v4, v12, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[19:20], s[4:5], v0, v8, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v4, v13, v[17:18]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v12, v[17:18]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v19, v16, 0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v5, v13
; GFX7-GISEL-NEXT: v_mov_b32_e32 v13, v20
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v0, v9, v[13:14]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v19, v4, v[5:6]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[19:20], s[4:5], v6, v14, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v1, v8, v[17:18]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v2, v10, 0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v0, v20
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v15, v[0:1]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v6, v18
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[20:21], s[4:5], v2, v11, v[6:7]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v14, v[0:1]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v17, v19, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v10, v[20:21]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v1, v7
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v0, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v6, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v2, v19, v[9:10]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v16, v[4:5]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v12, v2, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v3, v6, v[1:2]
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v8i64:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u32 v11, v2, v11
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v2, v10, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v10, v3, v10
; GFX8-SDAG-NEXT: v_mul_lo_u32 v15, v6, v15
; GFX8-SDAG-NEXT: v_add_u32_e32 v11, vcc, v17, v11
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v6, v14, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v10, vcc, v11, v10
; GFX8-SDAG-NEXT: v_mul_lo_u32 v11, v7, v14
; GFX8-SDAG-NEXT: v_add_u32_e32 v3, vcc, v3, v15
; GFX8-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v0, v8, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v11, vcc, v3, v11
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v4, v13
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v12, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v8, v1, v8
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v5, v12
; GFX8-SDAG-NEXT: v_add_u32_e32 v0, vcc, v4, v0
; GFX8-SDAG-NEXT: v_add_u32_e32 v7, vcc, v7, v9
; GFX8-SDAG-NEXT: v_add_u32_e32 v0, vcc, v0, v1
; GFX8-SDAG-NEXT: v_mul_lo_u32 v4, v6, v0
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v3, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v6, vcc, v7, v8
; GFX8-SDAG-NEXT: v_add_u32_e32 v7, vcc, v1, v4
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v16, v11
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v16, v2, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v10, v2
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v6, v3
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v5, v1
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v0, v1
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v4, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v3, vcc, v7, v3
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v3, v4
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v3
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v8i64:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v4, v12, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[19:20], s[4:5], v0, v8, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v4, v13, v[17:18]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v12, v[17:18]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v19, v16, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v5, v13
; GFX8-GISEL-NEXT: v_mov_b32_e32 v13, v20
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v0, v9, v[13:14]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v19, v4, v[5:6]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[19:20], s[4:5], v6, v14, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v1, v8, v[17:18]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v2, v10, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, v20
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v15, v[0:1]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v6, v18
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[20:21], s[4:5], v2, v11, v[6:7]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v7, v14, v[0:1]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v17, v19, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v10, v[20:21]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, v7
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v17, v0, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v6, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v2, v19, v[9:10]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v8, v16, v[4:5]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v12, v2, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v3, v6, v[1:2]
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v8i64:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u32 v16, v5, v12
; GFX9-SDAG-NEXT: v_mul_lo_u32 v13, v4, v13
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v4, v12, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v12, v1, v8
; GFX9-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v8, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v9, v12
; GFX9-SDAG-NEXT: v_mul_lo_u32 v8, v7, v14
; GFX9-SDAG-NEXT: v_mul_lo_u32 v9, v6, v15
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v6, v14, 0
; GFX9-SDAG-NEXT: v_add3_u32 v7, v7, v9, v8
; GFX9-SDAG-NEXT: v_mul_lo_u32 v8, v3, v10
; GFX9-SDAG-NEXT: v_mul_lo_u32 v9, v2, v11
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v2, v10, 0
; GFX9-SDAG-NEXT: v_add3_u32 v5, v5, v13, v16
; GFX9-SDAG-NEXT: v_add3_u32 v3, v3, v9, v8
; GFX9-SDAG-NEXT: v_mul_lo_u32 v8, v3, v6
; GFX9-SDAG-NEXT: v_mul_lo_u32 v7, v2, v7
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v2, v6, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v6, v1, v4
; GFX9-SDAG-NEXT: v_mul_lo_u32 v5, v0, v5
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v4, 0
; GFX9-SDAG-NEXT: v_add3_u32 v3, v3, v7, v8
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v5, v6
; GFX9-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v2, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v8i64:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[18:19], s[0:1], v0, v9, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v0, v8, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v1, v8, v[18:19]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v2, v11, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v17, v17, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v10, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v3, v10, v[8:9]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v4, v13, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v10, v1, v2
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v4, v12, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v5, v12, v[8:9]
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, v3, v4
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v6, v15, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v6, v14, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v7, v14, v[8:9]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v16, v1, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v5, v5, v6
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v16, v2, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v17, v2, v[8:9]
; GFX9-GISEL-NEXT: v_add_u32_e32 v7, v7, v2
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v0, v4, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v5, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v10, v4, v[0:1]
; GFX9-GISEL-NEXT: v_add_u32_e32 v3, v3, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v6, v3, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v6, v2, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v7, v2, v[4:5]
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v8i64:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u32 v16, v5, v12
; GFX10-SDAG-NEXT: v_mul_lo_u32 v13, v4, v13
; GFX10-SDAG-NEXT: v_mul_lo_u32 v17, v1, v8
; GFX10-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v8, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v18, v7, v14
; GFX10-SDAG-NEXT: v_mul_lo_u32 v15, v6, v15
; GFX10-SDAG-NEXT: v_mul_lo_u32 v19, v3, v10
; GFX10-SDAG-NEXT: v_mul_lo_u32 v11, v2, v11
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[2:3], s4, v2, v10, 0
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[5:6], s4, v6, v14, 0
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[7:8], s4, v4, v12, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v9, v17
; GFX10-SDAG-NEXT: v_add3_u32 v3, v3, v11, v19
; GFX10-SDAG-NEXT: v_add3_u32 v4, v6, v15, v18
; GFX10-SDAG-NEXT: v_add3_u32 v6, v8, v13, v16
; GFX10-SDAG-NEXT: v_mul_lo_u32 v8, v3, v5
; GFX10-SDAG-NEXT: v_mul_lo_u32 v9, v1, v7
; GFX10-SDAG-NEXT: v_mul_lo_u32 v4, v2, v4
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[2:3], s4, v2, v5, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v6, v0, v6
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v7, 0
; GFX10-SDAG-NEXT: v_add3_u32 v3, v3, v4, v8
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v6, v9
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX10-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v2, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v8i64:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[17:18], s4, v0, v8, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[19:20], s4, v2, v10, 0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v16, v1
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[21:22], s4, v6, v14, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[23:24], s4, v4, v12, 0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v18
; GFX10-GISEL-NEXT: v_mov_b32_e32 v18, v20
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[25:26], s4, v0, v9, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v2, v11, v[18:19]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v22
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v24
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[29:30], s4, v4, v13, v[2:3]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[26:27], s4, v6, v15, v[0:1]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[27:28], s4, v19, v21, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[30:31], s4, v17, v23, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], s4, v3, v10, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[6:7], s4, v7, v14, v[26:27]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v28
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], s4, v5, v12, v[29:30]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v4, v31
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[5:6], s4, v19, v6, v[0:1]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[6:7], s4, v16, v8, v[25:26]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], s4, v30, v27, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], s4, v17, v3, v[4:5]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], s4, v2, v21, v[5:6]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], s4, v6, v23, v[3:4]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], s4, v30, v4, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v2, v27, v[3:4]
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: test_vector_reduce_mul_v8i64:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v18, v5, v12
; GFX11-SDAG-NEXT: v_mul_lo_u32 v13, v4, v13
; GFX11-SDAG-NEXT: v_mul_lo_u32 v19, v1, v8
; GFX11-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[16:17], null, v0, v8, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v7, v14
; GFX11-SDAG-NEXT: v_mul_lo_u32 v8, v6, v15
; GFX11-SDAG-NEXT: v_mul_lo_u32 v15, v3, v10
; GFX11-SDAG-NEXT: v_mul_lo_u32 v11, v2, v11
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v2, v10, 0
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v6, v14, 0
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v4, v12, 0
; GFX11-SDAG-NEXT: v_add3_u32 v4, v17, v9, v19
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v11, v15
; GFX11-SDAG-NEXT: v_add3_u32 v3, v3, v8, v7
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-SDAG-NEXT: v_add3_u32 v6, v6, v13, v18
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v2
; GFX11-SDAG-NEXT: v_mul_lo_u32 v8, v4, v5
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v0, v3
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[3:4], null, v16, v5, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v9, v16, v6
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[5:6], null, v0, v2, 0
; GFX11-SDAG-NEXT: v_add3_u32 v0, v4, v9, v8
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_add3_u32 v1, v6, v7, v1
; GFX11-SDAG-NEXT: v_mul_lo_u32 v2, v0, v5
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v3, v1
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v3, v5, 0
; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v4, v2
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v8i64:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[17:18], null, v0, v8, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[19:20], null, v2, v10, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[21:22], null, v6, v14, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[23:24], null, v4, v12, 0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_dual_mov_b32 v16, v1 :: v_dual_mov_b32 v1, v18
; GFX11-GISEL-NEXT: v_mov_b32_e32 v18, v20
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[25:26], null, v0, v9, v[1:2]
; GFX11-GISEL-NEXT: v_dual_mov_b32 v0, v22 :: v_dual_mov_b32 v1, v24
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[26:27], null, v2, v11, v[18:19]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[27:28], null, v6, v15, v[0:1]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[30:31], null, v4, v13, v[1:2]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[28:29], null, v19, v21, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[31:32], null, v17, v23, 0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_3)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v7, v14, v[27:28]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v3, v10, v[26:27]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v5, v12, v[30:31]
; GFX11-GISEL-NEXT: v_mov_b32_e32 v1, v29
; GFX11-GISEL-NEXT: v_mov_b32_e32 v3, v32
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[9:10], null, v16, v8, v[25:26]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v19, v0, v[1:2]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v31, v28, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v17, v2, v[3:4]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v6, v21, v[4:5]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v9, v23, v[7:8]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v31, v2, v[1:2]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v3, v28, v[4:5]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-LABEL: test_vector_reduce_mul_v8i64:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v16, v5, v12
; GFX12-SDAG-NEXT: v_mul_lo_u32 v13, v4, v13
; GFX12-SDAG-NEXT: v_mul_lo_u32 v17, v1, v8
; GFX12-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v8, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v18, v7, v14
; GFX12-SDAG-NEXT: v_mul_lo_u32 v15, v6, v15
; GFX12-SDAG-NEXT: v_mul_lo_u32 v19, v3, v10
; GFX12-SDAG-NEXT: v_mul_lo_u32 v11, v2, v11
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v2, v10, 0
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[5:6], null, v6, v14, 0
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[7:8], null, v4, v12, 0
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v9, v17
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v11, v19
; GFX12-SDAG-NEXT: v_add3_u32 v4, v6, v15, v18
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-SDAG-NEXT: v_add3_u32 v6, v8, v13, v16
; GFX12-SDAG-NEXT: v_mul_lo_u32 v8, v3, v5
; GFX12-SDAG-NEXT: v_mul_lo_u32 v9, v1, v7
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v2, v4
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v2, v5, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v0, v6
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v7, 0
; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v4, v8
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v6, v9
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v0, v3
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v1, v2
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v2, 0
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v3, v4
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v8i64:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: v_mul_hi_u32 v16, v0, v8
; GFX12-GISEL-NEXT: v_mul_hi_u32 v17, v6, v14
; GFX12-GISEL-NEXT: v_mul_lo_u32 v21, v0, v8
; GFX12-GISEL-NEXT: v_mul_lo_u32 v22, v2, v10
; GFX12-GISEL-NEXT: v_mul_lo_u32 v23, v6, v14
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_2) | instid1(VALU_DEP_3)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[18:19], null, v0, v9, v[16:17]
; GFX12-GISEL-NEXT: v_mul_hi_u32 v9, v2, v10
; GFX12-GISEL-NEXT: v_mul_hi_u32 v0, v4, v12
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[15:16], null, v6, v15, v[17:18]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[16:17], null, v2, v11, v[9:10]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[19:20], null, v4, v13, v[0:1]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v9, v4, v12
; GFX12-GISEL-NEXT: v_mul_hi_u32 v0, v22, v23
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[6:7], null, v7, v14, v[15:16]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[2:3], null, v3, v10, v[16:17]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[3:4], null, v5, v12, v[19:20]
; GFX12-GISEL-NEXT: v_mul_hi_u32 v4, v21, v9
; GFX12-GISEL-NEXT: v_mul_lo_u32 v7, v21, v9
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[5:6], null, v22, v6, v[0:1]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v6, v22, v23
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[0:1], null, v1, v8, v[18:19]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[3:4], null, v21, v3, v[4:5]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v2, v23, v[5:6]
; GFX12-GISEL-NEXT: v_mul_hi_u32 v2, v7, v6
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[3:4], null, v0, v9, v[3:4]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v7, v6
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v7, v1, v[2:3]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v3, v6, v[1:2]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v8i64(<8 x i64> %v)
ret i64 %res
}
define i64 @test_vector_reduce_mul_v16i64(<16 x i64> %v) {
; GFX7-SDAG-LABEL: test_vector_reduce_mul_v16i64:
; GFX7-SDAG: ; %bb.0: ; %entry
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX7-SDAG-NEXT: v_mul_lo_u32 v19, v2, v19
; GFX7-SDAG-NEXT: v_mul_lo_u32 v32, v3, v18
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v2, v18, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v27, v10, v27
; GFX7-SDAG-NEXT: v_mul_lo_u32 v33, v11, v26
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v10, v26, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v3, vcc, v3, v19
; GFX7-SDAG-NEXT: v_mul_lo_u32 v15, v15, v30
; GFX7-SDAG-NEXT: v_add_i32_e32 v32, vcc, v3, v32
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v14, v30, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v30, v4, v21
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v20, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v5, v5, v20
; GFX7-SDAG-NEXT: v_mul_lo_u32 v29, v12, v29
; GFX7-SDAG-NEXT: v_mul_lo_u32 v34, v13, v28
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v12, v28, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v11, vcc, v11, v27
; GFX7-SDAG-NEXT: v_add_i32_e32 v11, vcc, v11, v33
; GFX7-SDAG-NEXT: v_add_i32_e32 v4, vcc, v4, v30
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[20:21], s[4:5], v2, v10, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v11
; GFX7-SDAG-NEXT: v_add_i32_e32 v11, vcc, v13, v29
; GFX7-SDAG-NEXT: v_add_i32_e32 v13, vcc, v4, v5
; GFX7-SDAG-NEXT: v_mul_lo_u32 v23, v6, v23
; GFX7-SDAG-NEXT: v_mul_lo_u32 v26, v7, v22
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v6, v22, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v17, v0, v17
; GFX7-SDAG-NEXT: v_mul_lo_u32 v10, v32, v10
; GFX7-SDAG-NEXT: v_add_i32_e32 v7, vcc, v7, v23
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v6, v18, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v7, vcc, v7, v26
; GFX7-SDAG-NEXT: v_add_i32_e32 v2, vcc, v21, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v7, v7, v18
; GFX7-SDAG-NEXT: v_add_i32_e32 v11, vcc, v11, v34
; GFX7-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX7-SDAG-NEXT: v_mul_lo_u32 v4, v14, v31
; GFX7-SDAG-NEXT: v_mul_lo_u32 v14, v1, v16
; GFX7-SDAG-NEXT: v_add_i32_e32 v4, vcc, v19, v4
; GFX7-SDAG-NEXT: v_add_i32_e32 v4, vcc, v4, v15
; GFX7-SDAG-NEXT: v_mul_lo_u32 v6, v6, v4
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v16, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v15, v8, v25
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v24, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v8, v9, v24
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, v5, v17
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v15
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v8
; GFX7-SDAG-NEXT: v_add_i32_e32 v8, vcc, v2, v10
; GFX7-SDAG-NEXT: v_mul_lo_u32 v9, v4, v1
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v4, v0, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v6, vcc, v23, v6
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, v5, v14
; GFX7-SDAG-NEXT: v_add_i32_e32 v4, vcc, v6, v7
; GFX7-SDAG-NEXT: v_add_i32_e32 v6, vcc, v2, v9
; GFX7-SDAG-NEXT: v_mul_lo_u32 v7, v3, v11
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v12, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v5, v5, v0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v0, v13, v12
; GFX7-SDAG-NEXT: v_add_i32_e32 v3, vcc, v3, v7
; GFX7-SDAG-NEXT: v_add_i32_e32 v5, vcc, v6, v5
; GFX7-SDAG-NEXT: v_add_i32_e32 v0, vcc, v3, v0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v3, v1, v0
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v1, v2, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v5, v2
; GFX7-SDAG-NEXT: v_mul_lo_u32 v5, v8, v22
; GFX7-SDAG-NEXT: v_add_i32_e32 v6, vcc, v1, v3
; GFX7-SDAG-NEXT: v_mul_lo_u32 v1, v20, v4
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v20, v22, 0
; GFX7-SDAG-NEXT: v_add_i32_e32 v2, vcc, v6, v2
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v4, v1
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v5
; GFX7-SDAG-NEXT: v_mul_lo_u32 v4, v0, v1
; GFX7-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v3, 0
; GFX7-SDAG-NEXT: v_mul_lo_u32 v2, v2, v3
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v4
; GFX7-SDAG-NEXT: v_add_i32_e32 v1, vcc, v1, v2
; GFX7-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX7-GISEL-LABEL: test_vector_reduce_mul_v16i64:
; GFX7-GISEL: ; %bb.0: ; %entry
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[31:32], s[4:5], v8, v24, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[32:33], s[4:5], v8, v25, v[32:33]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v9, v24, v[32:33]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v0, v16, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[32:33], s[4:5], v24, v31, 0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v9, v33
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v24, v8, v[9:10]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v24, v25
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v0, v17, v[24:25]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v1, v16, v[24:25]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v2, v18, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v31, v[8:9]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v26, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v10, v27, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v11, v26, v[9:10]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v16, v0, 0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v1, v11
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v16, v9, v[1:2]
; GFX7-GISEL-NEXT: buffer_load_dword v9, off, s[0:3], s32
; GFX7-GISEL-NEXT: v_mov_b32_e32 v1, v17
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v2, v19, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v3, v18, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v28, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v1, v0, v[24:25]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v4, v20, 0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v29, v[0:1]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v2, 0
; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, v18
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v28, v[0:1]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v21, v[3:4]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v1, v12
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v14, v30, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v20, v[3:4]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v17, v0, v[1:2]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v4, v13
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v6, v22, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v2, v[0:1]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v3, v18
; GFX7-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v9, v[4:5]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v12, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v15, v30, v[4:5]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v23, v[3:4]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v2, v14
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v17, v1, v[2:3]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v22, v[3:4]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v32, v11, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v3, v12, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v13, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v32, v0, v[5:6]
; GFX7-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v10, v1, v[0:1]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v16, v13, v[9:10]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v8, v11, v[5:6]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v9, v[1:2]
; GFX7-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v2, v[3:4]
; GFX7-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-SDAG-LABEL: test_vector_reduce_mul_v16i64:
; GFX8-SDAG: ; %bb.0: ; %entry
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX8-SDAG-NEXT: v_mul_lo_u32 v19, v2, v19
; GFX8-SDAG-NEXT: v_mul_lo_u32 v32, v3, v18
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v2, v18, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v27, v10, v27
; GFX8-SDAG-NEXT: v_mul_lo_u32 v33, v11, v26
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v10, v26, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v3, vcc, v3, v19
; GFX8-SDAG-NEXT: v_mul_lo_u32 v15, v15, v30
; GFX8-SDAG-NEXT: v_add_u32_e32 v32, vcc, v3, v32
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[18:19], s[4:5], v14, v30, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v30, v4, v21
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v20, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v5, v5, v20
; GFX8-SDAG-NEXT: v_mul_lo_u32 v29, v12, v29
; GFX8-SDAG-NEXT: v_mul_lo_u32 v34, v13, v28
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v12, v28, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v11, vcc, v11, v27
; GFX8-SDAG-NEXT: v_add_u32_e32 v11, vcc, v11, v33
; GFX8-SDAG-NEXT: v_add_u32_e32 v4, vcc, v4, v30
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[20:21], s[4:5], v2, v10, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v2, v11
; GFX8-SDAG-NEXT: v_add_u32_e32 v11, vcc, v13, v29
; GFX8-SDAG-NEXT: v_add_u32_e32 v13, vcc, v4, v5
; GFX8-SDAG-NEXT: v_mul_lo_u32 v23, v6, v23
; GFX8-SDAG-NEXT: v_mul_lo_u32 v26, v7, v22
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v6, v22, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v17, v0, v17
; GFX8-SDAG-NEXT: v_mul_lo_u32 v10, v32, v10
; GFX8-SDAG-NEXT: v_add_u32_e32 v7, vcc, v7, v23
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[22:23], s[4:5], v6, v18, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v7, vcc, v7, v26
; GFX8-SDAG-NEXT: v_add_u32_e32 v2, vcc, v21, v2
; GFX8-SDAG-NEXT: v_mul_lo_u32 v7, v7, v18
; GFX8-SDAG-NEXT: v_add_u32_e32 v11, vcc, v11, v34
; GFX8-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX8-SDAG-NEXT: v_mul_lo_u32 v4, v14, v31
; GFX8-SDAG-NEXT: v_mul_lo_u32 v14, v1, v16
; GFX8-SDAG-NEXT: v_add_u32_e32 v4, vcc, v19, v4
; GFX8-SDAG-NEXT: v_add_u32_e32 v4, vcc, v4, v15
; GFX8-SDAG-NEXT: v_mul_lo_u32 v6, v6, v4
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v0, v16, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v15, v8, v25
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v8, v24, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v8, v9, v24
; GFX8-SDAG-NEXT: v_add_u32_e32 v5, vcc, v5, v17
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v15
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v8
; GFX8-SDAG-NEXT: v_add_u32_e32 v8, vcc, v2, v10
; GFX8-SDAG-NEXT: v_mul_lo_u32 v9, v4, v1
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v4, v0, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v6, vcc, v23, v6
; GFX8-SDAG-NEXT: v_add_u32_e32 v5, vcc, v5, v14
; GFX8-SDAG-NEXT: v_add_u32_e32 v4, vcc, v6, v7
; GFX8-SDAG-NEXT: v_add_u32_e32 v6, vcc, v2, v9
; GFX8-SDAG-NEXT: v_mul_lo_u32 v7, v3, v11
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v3, v12, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v5, v5, v0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v0, v13, v12
; GFX8-SDAG-NEXT: v_add_u32_e32 v3, vcc, v3, v7
; GFX8-SDAG-NEXT: v_add_u32_e32 v5, vcc, v6, v5
; GFX8-SDAG-NEXT: v_add_u32_e32 v0, vcc, v3, v0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v3, v1, v0
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v1, v2, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v5, v2
; GFX8-SDAG-NEXT: v_mul_lo_u32 v5, v8, v22
; GFX8-SDAG-NEXT: v_add_u32_e32 v6, vcc, v1, v3
; GFX8-SDAG-NEXT: v_mul_lo_u32 v1, v20, v4
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v20, v22, 0
; GFX8-SDAG-NEXT: v_add_u32_e32 v2, vcc, v6, v2
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v4, v1
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v5
; GFX8-SDAG-NEXT: v_mul_lo_u32 v4, v0, v1
; GFX8-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v0, v3, 0
; GFX8-SDAG-NEXT: v_mul_lo_u32 v2, v2, v3
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v4
; GFX8-SDAG-NEXT: v_add_u32_e32 v1, vcc, v1, v2
; GFX8-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX8-GISEL-LABEL: test_vector_reduce_mul_v16i64:
; GFX8-GISEL: ; %bb.0: ; %entry
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[31:32], s[4:5], v8, v24, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[32:33], s[4:5], v8, v25, v[32:33]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v9, v24, v[32:33]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v0, v16, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[32:33], s[4:5], v24, v31, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v9, v33
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v24, v8, v[9:10]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v24, v25
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v0, v17, v[24:25]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v1, v16, v[24:25]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v2, v18, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v0, v31, v[8:9]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v10, v26, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v10, v27, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v11, v26, v[9:10]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v16, v0, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, v11
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[24:25], s[4:5], v16, v9, v[1:2]
; GFX8-GISEL-NEXT: buffer_load_dword v9, off, s[0:3], s32
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, v17
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v2, v19, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v3, v18, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v12, v28, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v1, v0, v[24:25]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v4, v20, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v12, v29, v[0:1]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v17, v2, 0
; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, v18
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v28, v[0:1]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v21, v[3:4]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v1, v12
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v14, v30, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v20, v[3:4]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v17, v0, v[1:2]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v4, v13
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[17:18], s[4:5], v6, v22, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v2, v[0:1]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v3, v18
; GFX8-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v14, v9, v[4:5]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[13:14], s[4:5], v17, v12, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v15, v30, v[4:5]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v6, v23, v[3:4]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v2, v14
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v17, v1, v[2:3]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v7, v22, v[3:4]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v32, v11, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v3, v12, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v13, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v32, v0, v[5:6]
; GFX8-GISEL-NEXT: v_mov_b32_e32 v0, v3
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v10, v1, v[0:1]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v4, v2, 0
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v16, v13, v[9:10]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v8, v11, v[5:6]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v4, v9, v[1:2]
; GFX8-GISEL-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v5, v2, v[3:4]
; GFX8-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-SDAG-LABEL: test_vector_reduce_mul_v16i64:
; GFX9-SDAG: ; %bb.0: ; %entry
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-SDAG-NEXT: scratch_load_dword v31, off, s32
; GFX9-SDAG-NEXT: v_mul_lo_u32 v32, v11, v26
; GFX9-SDAG-NEXT: v_mul_lo_u32 v27, v10, v27
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[10:11], s[0:1], v10, v26, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v26, v3, v18
; GFX9-SDAG-NEXT: v_mul_lo_u32 v33, v2, v19
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v2, v18, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v15, v15, v30
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[18:19], s[0:1], v14, v30, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v30, v7, v22
; GFX9-SDAG-NEXT: v_mul_lo_u32 v23, v6, v23
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v6, v22, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v22, v9, v24
; GFX9-SDAG-NEXT: v_mul_lo_u32 v25, v8, v25
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v8, v24, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v24, v1, v16
; GFX9-SDAG-NEXT: v_mul_lo_u32 v17, v0, v17
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v16, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v16, v13, v28
; GFX9-SDAG-NEXT: v_mul_lo_u32 v29, v12, v29
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[12:13], s[0:1], v12, v28, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v28, v5, v20
; GFX9-SDAG-NEXT: v_mul_lo_u32 v21, v4, v21
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v4, v20, 0
; GFX9-SDAG-NEXT: v_add3_u32 v9, v9, v25, v22
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v17, v24
; GFX9-SDAG-NEXT: v_add3_u32 v13, v13, v29, v16
; GFX9-SDAG-NEXT: v_add3_u32 v5, v5, v21, v28
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v4, v12, 0
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[20:21], s[0:1], v0, v8, 0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v5, v5, v12
; GFX9-SDAG-NEXT: v_mul_lo_u32 v4, v4, v13
; GFX9-SDAG-NEXT: v_mul_lo_u32 v1, v1, v8
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v0, v9
; GFX9-SDAG-NEXT: v_add3_u32 v4, v17, v4, v5
; GFX9-SDAG-NEXT: v_add3_u32 v5, v21, v0, v1
; GFX9-SDAG-NEXT: v_add3_u32 v7, v7, v23, v30
; GFX9-SDAG-NEXT: v_add3_u32 v11, v11, v27, v32
; GFX9-SDAG-NEXT: v_add3_u32 v3, v3, v33, v26
; GFX9-SDAG-NEXT: v_mul_lo_u32 v7, v7, v18
; GFX9-SDAG-NEXT: v_mul_lo_u32 v5, v5, v16
; GFX9-SDAG-NEXT: v_mul_lo_u32 v4, v20, v4
; GFX9-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX9-SDAG-NEXT: v_mul_lo_u32 v0, v14, v31
; GFX9-SDAG-NEXT: v_add3_u32 v0, v19, v0, v15
; GFX9-SDAG-NEXT: v_mul_lo_u32 v8, v6, v0
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v6, v18, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v8, v7
; GFX9-SDAG-NEXT: v_mul_lo_u32 v6, v3, v10
; GFX9-SDAG-NEXT: v_mul_lo_u32 v7, v2, v11
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v2, v10, 0
; GFX9-SDAG-NEXT: v_add3_u32 v3, v3, v7, v6
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v3, v0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v6, v2, v1
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v0, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v6, v3
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v20, v16, 0
; GFX9-SDAG-NEXT: v_add3_u32 v3, v3, v4, v5
; GFX9-SDAG-NEXT: v_mul_lo_u32 v3, v3, v0
; GFX9-SDAG-NEXT: v_mul_lo_u32 v4, v2, v1
; GFX9-SDAG-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v0, 0
; GFX9-SDAG-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX9-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-GISEL-LABEL: test_vector_reduce_mul_v16i64:
; GFX9-GISEL: ; %bb.0: ; %entry
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX9-GISEL-NEXT: scratch_load_dword v31, off, s32
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[34:35], s[0:1], v0, v17, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[32:33], s[0:1], v0, v16, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v1, v16, v[34:35]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v2, v19, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v33, v33, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v18, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v3, v18, v[16:17]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v4, v21, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v18, v1, v2
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v4, v20, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v5, v20, v[16:17]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v6, v23, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v19, v3, v4
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v6, v22, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v7, v22, v[16:17]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v8, v25, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v20, v5, v6
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v8, v24, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v9, v24, v[16:17]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v10, v27, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, v7, v8
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v10, v26, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[10:11], s[0:1], v11, v26, v[16:17]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v12, v29, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v3, v9, v10
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[10:11], s[0:1], v12, v28, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[12:13], s[0:1], v13, v28, v[16:17]
; GFX9-GISEL-NEXT: v_add_u32_e32 v5, v11, v12
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[12:13], s[0:1], v14, v30, 0
; GFX9-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v14, v31, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[14:15], s[0:1], v15, v30, v[16:17]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[16:17], s[0:1], v32, v1, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v9, v13, v14
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[14:15], s[0:1], v32, v6, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v33, v6, v[16:17]
; GFX9-GISEL-NEXT: v_add_u32_e32 v11, v15, v6
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[6:7], s[0:1], v0, v8, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v0, v3, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v18, v8, v[0:1]
; GFX9-GISEL-NEXT: v_add_u32_e32 v7, v7, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v2, v10, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v2, v5, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v19, v10, v[2:3]
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v4, v12, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v4, v9, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v20, v12, v[4:5]
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v14, v1, 0
; GFX9-GISEL-NEXT: v_add_u32_e32 v3, v3, v4
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[4:5], s[0:1], v14, v0, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v11, v0, v[8:9]
; GFX9-GISEL-NEXT: v_add_u32_e32 v5, v5, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v6, v3, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[8:9], s[0:1], v6, v2, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v7, v2, v[0:1]
; GFX9-GISEL-NEXT: v_add_u32_e32 v2, v9, v0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v4, v2, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[0:1], s[0:1], v4, v8, 0
; GFX9-GISEL-NEXT: v_mad_u64_u32 v[2:3], s[0:1], v5, v8, v[2:3]
; GFX9-GISEL-NEXT: v_add_u32_e32 v1, v1, v2
; GFX9-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-SDAG-LABEL: test_vector_reduce_mul_v16i64:
; GFX10-SDAG: ; %bb.0: ; %entry
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-SDAG-NEXT: buffer_load_dword v31, off, s[0:3], s32
; GFX10-SDAG-NEXT: v_mul_lo_u32 v32, v11, v26
; GFX10-SDAG-NEXT: v_mul_lo_u32 v27, v10, v27
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[10:11], s4, v10, v26, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v26, v3, v18
; GFX10-SDAG-NEXT: v_mul_lo_u32 v19, v2, v19
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[2:3], s4, v2, v18, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v18, v7, v22
; GFX10-SDAG-NEXT: v_mul_lo_u32 v23, v6, v23
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[6:7], s4, v6, v22, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v22, v9, v24
; GFX10-SDAG-NEXT: v_mul_lo_u32 v25, v8, v25
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[8:9], s4, v8, v24, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v24, v1, v16
; GFX10-SDAG-NEXT: v_mul_lo_u32 v17, v0, v17
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v0, v16, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v33, v15, v30
; GFX10-SDAG-NEXT: v_mul_lo_u32 v34, v13, v28
; GFX10-SDAG-NEXT: v_mul_lo_u32 v29, v12, v29
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[12:13], s4, v12, v28, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v28, v5, v20
; GFX10-SDAG-NEXT: v_mul_lo_u32 v21, v4, v21
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[4:5], s4, v4, v20, 0
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[15:16], s4, v14, v30, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v17, v24
; GFX10-SDAG-NEXT: v_add3_u32 v11, v11, v27, v32
; GFX10-SDAG-NEXT: v_add3_u32 v3, v3, v19, v26
; GFX10-SDAG-NEXT: v_add3_u32 v7, v7, v23, v18
; GFX10-SDAG-NEXT: v_add3_u32 v9, v9, v25, v22
; GFX10-SDAG-NEXT: v_add3_u32 v23, v13, v29, v34
; GFX10-SDAG-NEXT: v_add3_u32 v5, v5, v21, v28
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[19:20], s4, v0, v8, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v8, v1, v8
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[17:18], s4, v4, v12, 0
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[21:22], s4, v2, v10, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v5, v5, v12
; GFX10-SDAG-NEXT: v_mul_lo_u32 v4, v4, v23
; GFX10-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v3, v10
; GFX10-SDAG-NEXT: v_mul_lo_u32 v2, v2, v11
; GFX10-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX10-SDAG-NEXT: v_add3_u32 v4, v18, v4, v5
; GFX10-SDAG-NEXT: v_add3_u32 v8, v20, v9, v8
; GFX10-SDAG-NEXT: v_add3_u32 v5, v22, v2, v3
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[2:3], s4, v19, v17, 0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v4, v19, v4
; GFX10-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX10-SDAG-NEXT: v_mul_lo_u32 v24, v14, v31
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[13:14], s4, v6, v15, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v16, v24, v33
; GFX10-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX10-SDAG-NEXT: v_mul_lo_u32 v6, v6, v1
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v21, v13, 0
; GFX10-SDAG-NEXT: v_add3_u32 v6, v14, v6, v7
; GFX10-SDAG-NEXT: v_mul_lo_u32 v7, v8, v17
; GFX10-SDAG-NEXT: v_mul_lo_u32 v6, v21, v6
; GFX10-SDAG-NEXT: v_add3_u32 v3, v3, v4, v7
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v6, v5
; GFX10-SDAG-NEXT: v_mul_lo_u32 v3, v3, v0
; GFX10-SDAG-NEXT: v_mul_lo_u32 v4, v2, v1
; GFX10-SDAG-NEXT: v_mad_u64_u32 v[0:1], s4, v2, v0, 0
; GFX10-SDAG-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX10-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX10-GISEL-LABEL: test_vector_reduce_mul_v16i64:
; GFX10-GISEL: ; %bb.0: ; %entry
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[31:32], s4, v0, v16, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[33:34], s4, v2, v18, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[38:39], s4, v6, v22, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[35:36], s4, v0, v17, v[32:33]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[36:37], s4, v4, v20, 0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v34
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[16:17], s4, v1, v16, v[35:36]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], s4, v2, v19, v[0:1]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v37
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v39
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[34:35], s4, v8, v24, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[48:49], s4, v4, v21, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[49:50], s4, v10, v26, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v6, v23, v[2:3]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v35
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[5:6], s4, v5, v20, v[48:49]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[51:52], s4, v8, v25, v[2:3]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v50
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[52:53], s4, v10, v27, v[2:3]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[53:54], s4, v12, v28, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[9:10], s4, v9, v24, v[51:52]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v54
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[10:11], s4, v11, v26, v[52:53]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[19:20], s4, v36, v53, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[54:55], s4, v12, v29, v[2:3]
; GFX10-GISEL-NEXT: buffer_load_dword v12, off, s[0:3], s32
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], s4, v3, v18, v[0:1]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], s4, v14, v30, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[17:18], s4, v33, v49, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[6:7], s4, v7, v22, v[1:2]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v4
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[7:8], s4, v31, v34, 0
; GFX10-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], s4, v14, v12, v[0:1]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[11:12], s4, v38, v3, 0
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v8
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[13:14], s4, v13, v28, v[54:55]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[14:15], s4, v15, v30, v[0:1]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v18
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[8:9], s4, v31, v9, v[1:2]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v12
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[9:10], s4, v33, v10, v[0:1]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v0, v20
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[14:15], s4, v38, v14, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[20:21], s4, v17, v11, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], s4, v36, v13, v[0:1]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[12:13], s4, v7, v19, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], s4, v6, v3, v[14:15]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v1, v21
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[9:10], s4, v2, v49, v[9:10]
; GFX10-GISEL-NEXT: v_mov_b32_e32 v2, v13
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], s4, v5, v53, v[0:1]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[5:6], s4, v17, v3, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[0:1], s4, v12, v20, 0
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[2:3], s4, v7, v4, v[2:3]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[13:14], s4, v16, v34, v[8:9]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[3:4], s4, v9, v11, v[5:6]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[4:5], s4, v13, v19, v[2:3]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v12, v3, v[1:2]
; GFX10-GISEL-NEXT: v_mad_u64_u32 v[1:2], s4, v4, v20, v[1:2]
; GFX10-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-SDAG-LABEL: test_vector_reduce_mul_v16i64:
; GFX11-SDAG: ; %bb.0: ; %entry
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-SDAG-NEXT: scratch_load_b32 v33, off, s32
; GFX11-SDAG-NEXT: v_mul_lo_u32 v34, v11, v26
; GFX11-SDAG-NEXT: v_mul_lo_u32 v27, v10, v27
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[31:32], null, v10, v26, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v26, v3, v18
; GFX11-SDAG-NEXT: v_mul_lo_u32 v19, v2, v19
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[10:11], null, v2, v18, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v35, v15, v30
; GFX11-SDAG-NEXT: v_mul_lo_u32 v15, v7, v22
; GFX11-SDAG-NEXT: v_mul_lo_u32 v18, v6, v23
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v6, v22, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v22, v9, v24
; GFX11-SDAG-NEXT: v_mul_lo_u32 v23, v8, v25
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[6:7], null, v8, v24, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v24, v1, v16
; GFX11-SDAG-NEXT: v_mul_lo_u32 v17, v0, v17
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[8:9], null, v0, v16, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v16, v13, v28
; GFX11-SDAG-NEXT: v_mul_lo_u32 v25, v12, v29
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v12, v28, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v28, v5, v20
; GFX11-SDAG-NEXT: v_mul_lo_u32 v21, v4, v21
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[12:13], null, v4, v20, 0
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[4:5], null, v14, v30, 0
; GFX11-SDAG-NEXT: v_add3_u32 v7, v7, v23, v22
; GFX11-SDAG-NEXT: v_add3_u32 v27, v32, v27, v34
; GFX11-SDAG-NEXT: v_add3_u32 v11, v11, v19, v26
; GFX11-SDAG-NEXT: v_add3_u32 v3, v3, v18, v15
; GFX11-SDAG-NEXT: v_add3_u32 v9, v9, v17, v24
; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v25, v16
; GFX11-SDAG-NEXT: v_add3_u32 v21, v13, v21, v28
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[15:16], null, v12, v0, 0
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[17:18], null, v8, v6, 0
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[19:20], null, v10, v31, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v21, v0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v12, v1
; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v9, v6
; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v8, v7
; GFX11-SDAG-NEXT: v_mul_lo_u32 v8, v11, v31
; GFX11-SDAG-NEXT: v_mul_lo_u32 v9, v10, v27
; GFX11-SDAG-NEXT: v_add3_u32 v0, v16, v1, v0
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-SDAG-NEXT: v_add3_u32 v6, v18, v7, v6
; GFX11-SDAG-NEXT: v_add3_u32 v1, v20, v9, v8
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v0, v17, v0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v6, v6, v15
; GFX11-SDAG-NEXT: s_waitcnt vmcnt(0)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v22, v14, v33
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[13:14], null, v2, v4, 0
; GFX11-SDAG-NEXT: v_mul_lo_u32 v4, v3, v4
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX11-SDAG-NEXT: v_add3_u32 v5, v5, v22, v35
; GFX11-SDAG-NEXT: v_mul_lo_u32 v1, v1, v13
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v2, v5
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[2:3], null, v19, v13, 0
; GFX11-SDAG-NEXT: v_add3_u32 v7, v14, v5, v4
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[4:5], null, v17, v15, 0
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v7, v19, v7
; GFX11-SDAG-NEXT: v_add3_u32 v0, v5, v0, v6
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-SDAG-NEXT: v_add3_u32 v1, v3, v7, v1
; GFX11-SDAG-NEXT: v_mul_lo_u32 v3, v0, v2
; GFX11-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX11-SDAG-NEXT: v_mul_lo_u32 v5, v4, v1
; GFX11-SDAG-NEXT: v_mad_u64_u32 v[0:1], null, v4, v2, 0
; GFX11-SDAG-NEXT: v_add3_u32 v1, v1, v5, v3
; GFX11-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX11-GISEL-LABEL: test_vector_reduce_mul_v16i64:
; GFX11-GISEL: ; %bb.0: ; %entry
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0) expcnt(0) lgkmcnt(0)
; GFX11-GISEL-NEXT: scratch_load_b32 v71, off, s32
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[31:32], null, v0, v16, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[33:34], null, v2, v18, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[35:36], null, v4, v20, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[37:38], null, v6, v22, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[50:51], null, v10, v26, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[52:53], null, v12, v28, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[48:49], null, v8, v24, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[54:55], null, v14, v30, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[82:83], null, v0, v17, v[32:33]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[83:84], null, v2, v19, v[34:35]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[84:85], null, v4, v21, v[36:37]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[85:86], null, v6, v23, v[38:39]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[86:87], null, v10, v27, v[51:52]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[65:66], null, v31, v48, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[38:39], null, v8, v25, v[49:50]
; GFX11-GISEL-NEXT: v_mov_b32_e32 v64, v55
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[96:97], null, v12, v29, v[53:54]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[97:98], null, v1, v16, v[82:83]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v3, v18, v[83:84]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[2:3], null, v5, v20, v[84:85]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v7, v22, v[85:86]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[67:68], null, v33, v50, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[80:81], null, v37, v54, 0
; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v66
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[69:70], null, v35, v52, 0
; GFX11-GISEL-NEXT: s_waitcnt vmcnt(0)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v14, v71, v[64:65]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v9, v24, v[38:39]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v11, v26, v[86:87]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v13, v28, v[96:97]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_3) | instid1(VALU_DEP_3)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v15, v30, v[4:5]
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v68
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[9:10], null, v31, v5, v[0:1]
; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v81
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[10:11], null, v33, v6, v[4:5]
; GFX11-GISEL-NEXT: v_mov_b32_e32 v4, v70
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[5:6], null, v37, v8, v[0:1]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[11:12], null, v67, v80, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[13:14], null, v35, v7, v[4:5]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[6:7], null, v65, v69, 0
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_3) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mov_b32_e32 v0, v12
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[14:15], null, v3, v54, v[5:6]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[3:4], null, v1, v50, v[10:11]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_4) | instid1(VALU_DEP_4)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v2, v52, v[13:14]
; GFX11-GISEL-NEXT: v_mov_b32_e32 v2, v7
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v67, v14, v[0:1]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[12:13], null, v97, v48, v[9:10]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[0:1], null, v6, v11, 0
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[8:9], null, v65, v4, v[2:3]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[4:5], null, v3, v80, v[7:8]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[9:10], null, v12, v69, v[8:9]
; GFX11-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_1)
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[7:8], null, v6, v4, v[1:2]
; GFX11-GISEL-NEXT: v_mad_u64_u32 v[1:2], null, v9, v11, v[7:8]
; GFX11-GISEL-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-SDAG-LABEL: test_vector_reduce_mul_v16i64:
; GFX12-SDAG: ; %bb.0: ; %entry
; GFX12-SDAG-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-SDAG-NEXT: s_wait_expcnt 0x0
; GFX12-SDAG-NEXT: s_wait_samplecnt 0x0
; GFX12-SDAG-NEXT: s_wait_bvhcnt 0x0
; GFX12-SDAG-NEXT: s_wait_kmcnt 0x0
; GFX12-SDAG-NEXT: scratch_load_b32 v31, off, s32
; GFX12-SDAG-NEXT: v_mul_lo_u32 v32, v11, v26
; GFX12-SDAG-NEXT: v_mul_lo_u32 v27, v10, v27
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[10:11], null, v10, v26, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v26, v3, v18
; GFX12-SDAG-NEXT: v_mul_lo_u32 v19, v2, v19
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v2, v18, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v18, v7, v22
; GFX12-SDAG-NEXT: v_mul_lo_u32 v23, v6, v23
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[6:7], null, v6, v22, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v22, v9, v24
; GFX12-SDAG-NEXT: v_mul_lo_u32 v25, v8, v25
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[8:9], null, v8, v24, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v24, v1, v16
; GFX12-SDAG-NEXT: v_mul_lo_u32 v17, v0, v17
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v0, v16, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v33, v15, v30
; GFX12-SDAG-NEXT: v_mul_lo_u32 v34, v13, v28
; GFX12-SDAG-NEXT: v_mul_lo_u32 v29, v12, v29
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[12:13], null, v12, v28, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v28, v5, v20
; GFX12-SDAG-NEXT: v_mul_lo_u32 v21, v4, v21
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[4:5], null, v4, v20, 0
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[15:16], null, v14, v30, 0
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v17, v24
; GFX12-SDAG-NEXT: v_add3_u32 v11, v11, v27, v32
; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v19, v26
; GFX12-SDAG-NEXT: v_add3_u32 v7, v7, v23, v18
; GFX12-SDAG-NEXT: v_add3_u32 v9, v9, v25, v22
; GFX12-SDAG-NEXT: v_add3_u32 v23, v13, v29, v34
; GFX12-SDAG-NEXT: v_add3_u32 v5, v5, v21, v28
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[19:20], null, v0, v8, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v8, v1, v8
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[17:18], null, v4, v12, 0
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[21:22], null, v2, v10, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v5, v5, v12
; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v4, v23
; GFX12-SDAG-NEXT: v_mul_lo_u32 v9, v0, v9
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v3, v10
; GFX12-SDAG-NEXT: v_mul_lo_u32 v2, v2, v11
; GFX12-SDAG-NEXT: v_mul_lo_u32 v7, v7, v15
; GFX12-SDAG-NEXT: v_add3_u32 v4, v18, v4, v5
; GFX12-SDAG-NEXT: v_add3_u32 v8, v20, v9, v8
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(SKIP_1) | instid1(VALU_DEP_4)
; GFX12-SDAG-NEXT: v_add3_u32 v5, v22, v2, v3
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[2:3], null, v19, v17, 0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v19, v4
; GFX12-SDAG-NEXT: s_wait_loadcnt 0x0
; GFX12-SDAG-NEXT: v_mul_lo_u32 v24, v14, v31
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[13:14], null, v6, v15, 0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_add3_u32 v1, v16, v24, v33
; GFX12-SDAG-NEXT: v_mul_lo_u32 v5, v5, v13
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v6, v1
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v21, v13, 0
; GFX12-SDAG-NEXT: v_add3_u32 v6, v14, v6, v7
; GFX12-SDAG-NEXT: v_mul_lo_u32 v7, v8, v17
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v6, v21, v6
; GFX12-SDAG-NEXT: v_add3_u32 v3, v3, v4, v7
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v6, v5
; GFX12-SDAG-NEXT: v_mul_lo_u32 v3, v3, v0
; GFX12-SDAG-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_1)
; GFX12-SDAG-NEXT: v_mul_lo_u32 v4, v2, v1
; GFX12-SDAG-NEXT: v_mad_co_u64_u32 v[0:1], null, v2, v0, 0
; GFX12-SDAG-NEXT: v_add3_u32 v1, v1, v4, v3
; GFX12-SDAG-NEXT: s_setpc_b64 s[30:31]
;
; GFX12-GISEL-LABEL: test_vector_reduce_mul_v16i64:
; GFX12-GISEL: ; %bb.0: ; %entry
; GFX12-GISEL-NEXT: s_wait_loadcnt_dscnt 0x0
; GFX12-GISEL-NEXT: s_wait_expcnt 0x0
; GFX12-GISEL-NEXT: s_wait_samplecnt 0x0
; GFX12-GISEL-NEXT: s_wait_bvhcnt 0x0
; GFX12-GISEL-NEXT: s_wait_kmcnt 0x0
; GFX12-GISEL-NEXT: scratch_load_b32 v39, off, s32
; GFX12-GISEL-NEXT: v_mul_hi_u32 v31, v0, v16
; GFX12-GISEL-NEXT: v_mul_hi_u32 v32, v2, v18
; GFX12-GISEL-NEXT: v_mul_hi_u32 v33, v4, v20
; GFX12-GISEL-NEXT: v_mul_hi_u32 v34, v6, v22
; GFX12-GISEL-NEXT: v_mul_hi_u32 v35, v8, v24
; GFX12-GISEL-NEXT: v_mul_hi_u32 v36, v10, v26
; GFX12-GISEL-NEXT: v_mul_hi_u32 v37, v12, v28
; GFX12-GISEL-NEXT: v_mul_hi_u32 v38, v14, v30
; GFX12-GISEL-NEXT: v_mul_lo_u32 v51, v2, v18
; GFX12-GISEL-NEXT: v_mul_lo_u32 v53, v6, v22
; GFX12-GISEL-NEXT: v_mul_lo_u32 v55, v10, v26
; GFX12-GISEL-NEXT: v_mul_lo_u32 v65, v14, v30
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[48:49], null, v0, v17, v[31:32]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[31:32], null, v2, v19, v[32:33]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[32:33], null, v4, v21, v[33:34]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[33:34], null, v6, v23, v[34:35]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[34:35], null, v8, v25, v[35:36]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[35:36], null, v10, v27, v[36:37]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[36:37], null, v12, v29, v[37:38]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v50, v0, v16
; GFX12-GISEL-NEXT: v_mul_lo_u32 v52, v4, v20
; GFX12-GISEL-NEXT: v_mul_lo_u32 v54, v8, v24
; GFX12-GISEL-NEXT: v_mul_lo_u32 v64, v12, v28
; GFX12-GISEL-NEXT: v_mul_lo_u32 v23, v51, v55
; GFX12-GISEL-NEXT: v_mul_lo_u32 v27, v53, v65
; GFX12-GISEL-NEXT: v_mul_hi_u32 v6, v53, v65
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[7:8], null, v7, v22, v[33:34]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[16:17], null, v1, v16, v[48:49]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v21, v50, v54
; GFX12-GISEL-NEXT: v_mul_lo_u32 v25, v52, v64
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[8:9], null, v9, v24, v[34:35]
; GFX12-GISEL-NEXT: v_mul_hi_u32 v2, v51, v55
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[17:18], null, v3, v18, v[31:32]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[9:10], null, v11, v26, v[35:36]
; GFX12-GISEL-NEXT: v_mul_hi_u32 v3, v23, v27
; GFX12-GISEL-NEXT: v_mul_hi_u32 v4, v52, v64
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[18:19], null, v5, v20, v[32:33]
; GFX12-GISEL-NEXT: v_mul_hi_u32 v0, v50, v54
; GFX12-GISEL-NEXT: v_mul_hi_u32 v1, v21, v25
; GFX12-GISEL-NEXT: s_wait_loadcnt 0x0
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[37:38], null, v14, v39, v[38:39]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(SKIP_3) | instid1(VALU_DEP_4)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[14:15], null, v15, v30, v[37:38]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[10:11], null, v13, v28, v[36:37]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[11:12], null, v51, v9, v[2:3]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v13, v21, v25
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[5:6], null, v53, v14, v[6:7]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1) | instskip(NEXT) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[9:10], null, v52, v10, v[4:5]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[4:5], null, v7, v65, v[5:6]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[5:6], null, v50, v8, v[0:1]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[6:7], null, v17, v55, v[11:12]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[7:8], null, v18, v64, v[9:10]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[2:3], null, v23, v4, v[3:4]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v8, v23, v27
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_4) | instskip(NEXT) | instid1(VALU_DEP_3)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[3:4], null, v16, v54, v[5:6]
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[0:1], null, v21, v7, v[1:2]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(NEXT) | instid1(VALU_DEP_4)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v6, v27, v[2:3]
; GFX12-GISEL-NEXT: v_mul_hi_u32 v2, v13, v8
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_2) | instskip(SKIP_1) | instid1(VALU_DEP_2)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[3:4], null, v3, v25, v[0:1]
; GFX12-GISEL-NEXT: v_mul_lo_u32 v0, v13, v8
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v13, v1, v[2:3]
; GFX12-GISEL-NEXT: s_delay_alu instid0(VALU_DEP_1)
; GFX12-GISEL-NEXT: v_mad_co_u64_u32 v[1:2], null, v3, v8, v[1:2]
; GFX12-GISEL-NEXT: s_setpc_b64 s[30:31]
entry:
%res = call i64 @llvm.vector.reduce.mul.v16i64(<16 x i64> %v)
ret i64 %res
}
declare i8 @llvm.vector.reduce.mul.v2i8(<2 x i8>)
declare i8 @llvm.vector.reduce.mul.v3i8(<3 x i8>)
declare i8 @llvm.vector.reduce.mul.v4i8(<4 x i8>)
declare i8 @llvm.vector.reduce.mul.v8i8(<8 x i8>)
declare i8 @llvm.vector.reduce.mul.v16i8(<16 x i8>)
declare i16 @llvm.vector.reduce.mul.v2i16(<2 x i16>)
declare i16 @llvm.vector.reduce.mul.v3i16(<3 x i16>)
declare i16 @llvm.vector.reduce.mul.v4i16(<4 x i16>)
declare i16 @llvm.vector.reduce.mul.v8i16(<8 x i16>)
declare i16 @llvm.vector.reduce.mul.v16i16(<16 x i16>)
declare i32 @llvm.vector.reduce.mul.v2i32(<2 x i32>)
declare i32 @llvm.vector.reduce.mul.v3i32(<3 x i32>)
declare i32 @llvm.vector.reduce.mul.v4i32(<4 x i32>)
declare i32 @llvm.vector.reduce.mul.v8i32(<8 x i32>)
declare i32 @llvm.vector.reduce.mul.v16i32(<16 x i32>)
declare i64 @llvm.vector.reduce.mul.v2i64(<2 x i64>)
declare i64 @llvm.vector.reduce.mul.v3i64(<3 x i64>)
declare i64 @llvm.vector.reduce.mul.v4i64(<4 x i64>)
declare i64 @llvm.vector.reduce.mul.v8i64(<8 x i64>)
declare i64 @llvm.vector.reduce.mul.v16i64(<16 x i64>)
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
; GFX11-GISEL-FAKE16: {{.*}}
; GFX12-GISEL-FAKE16: {{.*}}
Reference in New Issue View Git Blame Copy Permalink