SelectionDAG: Do not propagate divergence through glue (#174766)

Glue does not carry any value (in the LLVM IR Value sense) that could be
considered uniform or divergent.
This commit is contained in:
Jay Foad 2026-01-07 14:04:36 +00:00 committed by GitHub
parent 82c1f9435d
commit 47a0d0e428
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
6 changed files with 978 additions and 972 deletions

View File

@ -12384,7 +12384,7 @@ void SelectionDAG::ReplaceAllUsesWith(SDNode *From, const SDValue *To) {
const SDValue &ToOp = To[Use.getResNo()];
++UI;
Use.set(ToOp);
if (ToOp.getValueType() != MVT::Other)
if (ToOp.getValueType() != MVT::Other && ToOp.getValueType() != MVT::Glue)
To_IsDivergent |= ToOp->isDivergent();
} while (UI != UE && UI->getUser() == User);
@ -12501,19 +12501,6 @@ public:
} // end anonymous namespace
/// Return true if a glue output should propagate divergence information.
static bool gluePropagatesDivergence(const SDNode *Node) {
switch (Node->getOpcode()) {
case ISD::CopyFromReg:
case ISD::CopyToReg:
return false;
default:
return true;
}
llvm_unreachable("covered opcode switch");
}
bool SelectionDAG::calculateDivergence(SDNode *N) {
if (TLI->isSDNodeAlwaysUniform(N)) {
assert(!TLI->isSDNodeSourceOfDivergence(N, FLI, UA) &&
@ -12525,9 +12512,8 @@ bool SelectionDAG::calculateDivergence(SDNode *N) {
for (const auto &Op : N->ops()) {
EVT VT = Op.getValueType();
// Skip Chain. It does not carry divergence.
if (VT != MVT::Other && Op.getNode()->isDivergent() &&
(VT != MVT::Glue || gluePropagatesDivergence(Op.getNode())))
// Skip Chain and Glue. They do not carry divergence.
if (VT != MVT::Other && VT != MVT::Glue && Op.getNode()->isDivergent())
return true;
}
return false;
@ -14159,12 +14145,9 @@ void SelectionDAG::createOperands(SDNode *Node, ArrayRef<SDValue> Vals) {
Ops[I].setInitial(Vals[I]);
EVT VT = Ops[I].getValueType();
// Skip Chain. It does not carry divergence.
if (VT != MVT::Other &&
(VT != MVT::Glue || gluePropagatesDivergence(Ops[I].getNode())) &&
Ops[I].getNode()->isDivergent()) {
// Skip Chain and Glue. They do not carry divergence.
if (VT != MVT::Other && VT != MVT::Glue && Ops[I].getNode()->isDivergent())
IsDivergent = true;
}
}
Node->NumOperands = Vals.size();
Node->OperandList = Ops;

View File

@ -80,12 +80,13 @@ define <2 x i128> @wide_carry_divergence_error(i128 %arg) {
; GCN-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GCN-NEXT: v_min_u32_e32 v4, v4, v5
; GCN-NEXT: v_cndmask_b32_e32 v0, v0, v4, vcc
; GCN-NEXT: s_subb_u32 s4, 0, 0
; GCN-NEXT: v_cndmask_b32_e64 v1, v1, 0, vcc
; GCN-NEXT: v_sub_u32_e32 v0, vcc, 0, v0
; GCN-NEXT: v_mov_b32_e32 v3, 0
; GCN-NEXT: s_subb_u32 s5, 0, 0
; GCN-NEXT: v_subb_u32_e32 v1, vcc, 0, v1, vcc
; GCN-NEXT: v_subb_u32_e32 v2, vcc, 0, v3, vcc
; GCN-NEXT: v_subb_u32_e32 v3, vcc, 0, v3, vcc
; GCN-NEXT: v_mov_b32_e32 v2, s4
; GCN-NEXT: v_mov_b32_e32 v3, s5
; GCN-NEXT: v_mov_b32_e32 v4, 0
; GCN-NEXT: v_mov_b32_e32 v5, 0
; GCN-NEXT: v_mov_b32_e32 v6, 0

View File

@ -24,7 +24,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, 0, v6, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, 0, v7, vcc
; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[6:7]
; GFX9-NEXT: v_ashrrev_i32_e32 v18, 31, v7
; GFX9-NEXT: s_mov_b64 s[8:9], 0x7f
; GFX9-NEXT: v_cndmask_b32_e32 v21, v5, v1, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v22, v4, v0, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v1, v7, v3, vcc
@ -43,57 +43,60 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_add_u32_e32 v3, 32, v3
; GFX9-NEXT: v_ffbh_u32_e32 v4, v21
; GFX9-NEXT: v_min_u32_e32 v3, v3, v4
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v3, vcc, 64, v3
; GFX9-NEXT: v_addc_co_u32_e64 v4, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v4, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[0:1]
; GFX9-NEXT: v_ffbh_u32_e32 v6, v11
; GFX9-NEXT: v_ffbh_u32_e32 v5, v11
; GFX9-NEXT: v_cndmask_b32_e32 v2, v3, v2, vcc
; GFX9-NEXT: v_ffbh_u32_e32 v3, v10
; GFX9-NEXT: v_add_u32_e32 v3, 32, v3
; GFX9-NEXT: v_min_u32_e32 v3, v3, v6
; GFX9-NEXT: v_ffbh_u32_e32 v6, v8
; GFX9-NEXT: v_add_u32_e32 v6, 32, v6
; GFX9-NEXT: v_ffbh_u32_e32 v7, v9
; GFX9-NEXT: v_min_u32_e32 v6, v6, v7
; GFX9-NEXT: v_min_u32_e32 v3, v3, v5
; GFX9-NEXT: v_ffbh_u32_e32 v5, v8
; GFX9-NEXT: v_add_u32_e32 v5, 32, v5
; GFX9-NEXT: v_ffbh_u32_e32 v6, v9
; GFX9-NEXT: v_min_u32_e32 v5, v5, v6
; GFX9-NEXT: v_cndmask_b32_e64 v4, v4, 0, vcc
; GFX9-NEXT: v_add_co_u32_e32 v6, vcc, 64, v6
; GFX9-NEXT: v_addc_co_u32_e64 v7, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_add_co_u32_e32 v5, vcc, 64, v5
; GFX9-NEXT: v_addc_co_u32_e64 v6, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX9-NEXT: v_mov_b32_e32 v5, 0
; GFX9-NEXT: v_cndmask_b32_e32 v3, v6, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v7, v7, 0, vcc
; GFX9-NEXT: s_subb_u32 s4, 0, 0
; GFX9-NEXT: v_cndmask_b32_e32 v3, v5, v3, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v6, v6, 0, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v2, vcc, v2, v3
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v7, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v4, vcc, 0, v5, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v5, vcc, 0, v5, vcc
; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v19, v17
; GFX9-NEXT: v_cndmask_b32_e64 v6, 0, 1, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v6, vcc
; GFX9-NEXT: s_subb_u32 s5, 0, 0
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[2:3]
; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-NEXT: s_cmp_eq_u64 s[4:5], 0
; GFX9-NEXT: v_cndmask_b32_e64 v4, 0, 1, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v5, 0, 1, s[8:9]
; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
; GFX9-NEXT: v_cndmask_b32_e32 v4, v5, v4, vcc
; GFX9-NEXT: v_and_b32_e32 v4, 1, v4
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v4
; GFX9-NEXT: v_xor_b32_e32 v4, 0x7f, v2
; GFX9-NEXT: v_or_b32_e32 v5, s5, v3
; GFX9-NEXT: v_or_b32_e32 v4, s4, v4
; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX9-NEXT: v_ashrrev_i32_e32 v18, 31, v7
; GFX9-NEXT: s_xor_b64 s[8:9], s[6:7], -1
; GFX9-NEXT: v_mov_b32_e32 v19, v17
; GFX9-NEXT: v_mov_b32_e32 v20, v18
; GFX9-NEXT: v_cndmask_b32_e64 v7, 0, 1, vcc
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[4:5]
; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
; GFX9-NEXT: v_and_b32_e32 v6, 1, v6
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v6
; GFX9-NEXT: v_xor_b32_e32 v6, 0x7f, v2
; GFX9-NEXT: v_or_b32_e32 v6, v6, v4
; GFX9-NEXT: v_or_b32_e32 v7, v3, v5
; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1
; GFX9-NEXT: v_cndmask_b32_e64 v13, v11, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v12, v10, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v7, v9, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v6, v8, 0, s[4:5]
; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v13, v11, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v12, v10, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v4, v9, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v5, v8, 0, s[6:7]
; GFX9-NEXT: s_and_b64 s[6:7], s[8:9], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[6:7]
; GFX9-NEXT: s_cbranch_execz .LBB0_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
; GFX9-NEXT: v_add_co_u32_e32 v23, vcc, 1, v2
; GFX9-NEXT: v_mov_b32_e32 v4, s4
; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v3, vcc
; GFX9-NEXT: v_mov_b32_e32 v5, s5
; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v4, vcc
; GFX9-NEXT: v_sub_u32_e32 v7, 0x7f, v2
; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v5, vcc
@ -197,20 +200,20 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_lshlrev_b64 v[0:1], 1, v[4:5]
; GFX9-NEXT: v_lshrrev_b32_e32 v2, 31, v5
; GFX9-NEXT: v_or_b32_e32 v12, v12, v2
; GFX9-NEXT: v_or_b32_e32 v7, v7, v1
; GFX9-NEXT: v_or_b32_e32 v6, v6, v0
; GFX9-NEXT: v_or_b32_e32 v4, v7, v1
; GFX9-NEXT: v_or_b32_e32 v5, v6, v0
; GFX9-NEXT: .LBB0_6: ; %Flow3
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: v_xor_b32_e32 v2, v18, v17
; GFX9-NEXT: v_xor_b32_e32 v3, v20, v19
; GFX9-NEXT: v_xor_b32_e32 v0, v6, v2
; GFX9-NEXT: v_xor_b32_e32 v1, v7, v3
; GFX9-NEXT: v_xor_b32_e32 v0, v5, v2
; GFX9-NEXT: v_xor_b32_e32 v1, v4, v3
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v2
; GFX9-NEXT: v_xor_b32_e32 v5, v12, v2
; GFX9-NEXT: v_xor_b32_e32 v7, v12, v2
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v3, vcc
; GFX9-NEXT: v_xor_b32_e32 v4, v13, v3
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v5, v2, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v4, v3, vcc
; GFX9-NEXT: v_xor_b32_e32 v6, v13, v3
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v7, v2, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v6, v3, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-O0-LABEL: v_sdiv_i128_vv:
@ -240,9 +243,7 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 0
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 1
; GFX9-O0-NEXT: s_mov_b32 s10, s6
; GFX9-O0-NEXT: v_writelane_b32 v30, s10, 2
; GFX9-O0-NEXT: s_mov_b32 s11, s7
; GFX9-O0-NEXT: v_writelane_b32 v30, s11, 3
; GFX9-O0-NEXT: v_sub_co_u32_e32 v5, vcc, s10, v1
; GFX9-O0-NEXT: v_mov_b32_e32 v0, s11
; GFX9-O0-NEXT: v_subb_co_u32_e32 v3, vcc, v0, v2, vcc
@ -416,49 +417,49 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr5_vgpr6 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9
; GFX9-O0-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v7
; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v5, v6, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s10
; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v5, v6, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s11
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11
; GFX9-O0-NEXT: v_subb_co_u32_e32 v6, vcc, v5, v6, vcc
; GFX9-O0-NEXT: s_subb_u32 s12, s10, s10
; GFX9-O0-NEXT: s_subb_u32 s8, s11, s11
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 def $sgpr12_sgpr13
; GFX9-O0-NEXT: s_mov_b32 s13, s8
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v8
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s12
; GFX9-O0-NEXT: v_mov_b32_e32 v7, s13
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
; GFX9-O0-NEXT: s_mov_b64 s[12:13], 0x7f
; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[14:15]
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[14:15]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[8:9]
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[14:15], 0x7f
; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[8:9], v[4:5], s[14:15]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[8:9]
; GFX9-O0-NEXT: s_cmp_lg_u64 s[12:13], s[6:7]
; GFX9-O0-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[8:9]
; GFX9-O0-NEXT: s_cmp_eq_u64 s[12:13], s[6:7]
; GFX9-O0-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9]
; GFX9-O0-NEXT: v_and_b32_e64 v6, 1, v6
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[8:9], v6, 1
; GFX9-O0-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GFX9-O0-NEXT: s_mov_b64 s[14:15], -1
; GFX9-O0-NEXT: s_mov_b64 s[16:17], -1
; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[4:5], s[14:15]
; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[4:5], s[16:17]
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5
; GFX9-O0-NEXT: s_mov_b32 s14, s13
; GFX9-O0-NEXT: v_xor_b32_e64 v6, v6, s14
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, s12
; GFX9-O0-NEXT: s_mov_b32 s16, s15
; GFX9-O0-NEXT: v_xor_b32_e64 v6, v6, s16
; GFX9-O0-NEXT: ; kill: def $sgpr14 killed $sgpr14 killed $sgpr14_sgpr15
; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, s14
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v8
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v9
; GFX9-O0-NEXT: s_mov_b32 s14, s13
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, s14
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v5
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, s12
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[4:5], s[6:7]
@ -485,8 +486,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 4
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 5
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 2
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 3
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
@ -499,8 +500,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 4
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 5
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: ; %bb.2: ; %Flow
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:116 ; 4-byte Folded Reload
@ -533,8 +534,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 4
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 5
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 3
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
@ -593,8 +594,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 8
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 9
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:108 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:112 ; 4-byte Folded Reload
@ -624,8 +625,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 10
; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 11
; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 8
; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 9
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:196 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:200 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:204 ; 4-byte Folded Reload
@ -799,11 +800,11 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:144 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 6
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 7
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 4
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 5
; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 10
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 11
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
@ -927,8 +928,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:268 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 10
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 11
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
@ -1077,8 +1078,8 @@ define i128 @v_sdiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 9
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 6
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 7
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
@ -2239,9 +2240,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_add_u32_e32 v9, 32, v9
; GFX9-NEXT: v_ffbh_u32_e32 v10, v5
; GFX9-NEXT: v_min_u32_e32 v9, v9, v10
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9
; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v10, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX9-NEXT: v_ffbh_u32_e32 v11, v3
; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
@ -2254,42 +2255,45 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_min_u32_e32 v11, v11, v12
; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, 64, v11
; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v12, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f
; GFX9-NEXT: s_subb_u32 s4, 0, 0
; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v13, v12, 0, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v12, vcc, v8, v9
; GFX9-NEXT: v_subb_co_u32_e32 v13, vcc, v10, v13, vcc
; GFX9-NEXT: v_mov_b32_e32 v8, 0
; GFX9-NEXT: v_subb_co_u32_e32 v14, vcc, 0, v8, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v15, vcc, 0, v8, vcc
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[12:13]
; GFX9-NEXT: v_or_b32_e32 v11, v13, v15
; GFX9-NEXT: s_subb_u32 s5, 0, 0
; GFX9-NEXT: s_mov_b64 s[8:9], 0x7f
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[12:13]
; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-NEXT: s_cmp_eq_u64 s[4:5], 0
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[14:15]
; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, 1, vcc
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[14:15]
; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[8:9]
; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
; GFX9-NEXT: v_and_b32_e32 v8, 1, v8
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8
; GFX9-NEXT: v_xor_b32_e32 v8, 0x7f, v12
; GFX9-NEXT: v_or_b32_e32 v10, v8, v14
; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
; GFX9-NEXT: v_or_b32_e32 v11, s5, v13
; GFX9-NEXT: v_or_b32_e32 v10, s4, v8
; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1
; GFX9-NEXT: v_cndmask_b32_e64 v9, v3, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v8, v2, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v10, v1, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v11, v0, 0, s[4:5]
; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
; GFX9-NEXT: s_xor_b64 s[8:9], s[6:7], -1
; GFX9-NEXT: v_cndmask_b32_e64 v9, v3, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v8, v2, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v10, v1, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v11, v0, 0, s[6:7]
; GFX9-NEXT: s_and_b64 s[6:7], s[8:9], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[6:7]
; GFX9-NEXT: s_cbranch_execz .LBB1_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
; GFX9-NEXT: v_mov_b32_e32 v9, s5
; GFX9-NEXT: v_add_co_u32_e32 v18, vcc, 1, v12
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_addc_co_u32_e32 v19, vcc, 0, v13, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v20, vcc, 0, v14, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v15, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v20, vcc, 0, v8, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v21, vcc, 0, v9, vcc
; GFX9-NEXT: v_sub_u32_e32 v15, 0x7f, v12
; GFX9-NEXT: v_or_b32_e32 v9, v19, v21
; GFX9-NEXT: v_or_b32_e32 v8, v18, v20
@ -2536,48 +2540,48 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b32 s10, s6
; GFX9-O0-NEXT: s_mov_b32 s11, s7
; GFX9-O0-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v7
; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v5, v6, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s10
; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v5, v6, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s11
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11
; GFX9-O0-NEXT: v_subb_co_u32_e32 v6, vcc, v5, v6, vcc
; GFX9-O0-NEXT: s_subb_u32 s12, s10, s10
; GFX9-O0-NEXT: s_subb_u32 s8, s11, s11
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 def $sgpr12_sgpr13
; GFX9-O0-NEXT: s_mov_b32 s13, s8
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v8
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s12
; GFX9-O0-NEXT: v_mov_b32_e32 v7, s13
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
; GFX9-O0-NEXT: s_mov_b64 s[12:13], 0x7f
; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[14:15]
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[14:15]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[8:9]
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[14:15], 0x7f
; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[8:9], v[4:5], s[14:15]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[8:9]
; GFX9-O0-NEXT: s_cmp_lg_u64 s[12:13], s[6:7]
; GFX9-O0-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[8:9]
; GFX9-O0-NEXT: s_cmp_eq_u64 s[12:13], s[6:7]
; GFX9-O0-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9]
; GFX9-O0-NEXT: v_and_b32_e64 v6, 1, v6
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[8:9], v6, 1
; GFX9-O0-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GFX9-O0-NEXT: s_mov_b64 s[4:5], -1
; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5
; GFX9-O0-NEXT: s_mov_b32 s14, s13
; GFX9-O0-NEXT: v_xor_b32_e64 v6, v6, s14
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, s12
; GFX9-O0-NEXT: s_mov_b32 s16, s15
; GFX9-O0-NEXT: v_xor_b32_e64 v6, v6, s16
; GFX9-O0-NEXT: ; kill: def $sgpr14 killed $sgpr14 killed $sgpr14_sgpr15
; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, s14
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v8
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v9
; GFX9-O0-NEXT: s_mov_b32 s14, s13
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, s14
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v5
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, s12
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[4:5], s[6:7]
@ -2603,17 +2607,17 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 2
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 3
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-O0-NEXT: s_cbranch_execz .LBB1_3
; GFX9-O0-NEXT: s_branch .LBB1_8
; GFX9-O0-NEXT: .LBB1_1: ; %Flow
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 4
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 5
@ -2645,9 +2649,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:76 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_5
; GFX9-O0-NEXT: .LBB1_3: ; %Flow2
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 3
@ -2705,9 +2709,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_3
; GFX9-O0-NEXT: .LBB1_5: ; %Flow1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
@ -2736,9 +2740,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_branch .LBB1_4
; GFX9-O0-NEXT: .LBB1_6: ; %udiv-do-while
; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 8
; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 9
@ -2920,9 +2924,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
@ -2951,9 +2955,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execnz .LBB1_6
; GFX9-O0-NEXT: s_branch .LBB1_1
; GFX9-O0-NEXT: .LBB1_7: ; %udiv-preheader
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:264 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:268 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:272 ; 4-byte Folded Reload
@ -3045,9 +3049,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:260 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:240 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:244 ; 4-byte Folded Spill
@ -3074,9 +3078,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:188 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_6
; GFX9-O0-NEXT: .LBB1_8: ; %udiv-bb1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@ -3195,9 +3199,9 @@ define i128 @v_udiv_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 6
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 7
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-O0-NEXT: s_cbranch_execz .LBB1_5
; GFX9-O0-NEXT: s_branch .LBB1_7

File diff suppressed because it is too large Load Diff

View File

@ -21,32 +21,33 @@ define i128 @fptosi_f64_to_i128(double %x) {
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: s_addc_u32 s6, 0, -1
; SDAG-NEXT: s_movk_i32 s10, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
; SDAG-NEXT: s_mov_b32 s7, -1
; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
; SDAG-NEXT: v_cmp_lt_i64_e32 vcc, -1, v[4:5]
; SDAG-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; SDAG-NEXT: s_addc_u32 s7, 0, -1
; SDAG-NEXT: s_mov_b32 s11, -1
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
; SDAG-NEXT: s_cmp_eq_u64 s[6:7], -1
; SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], -1, v[4:5]
; SDAG-NEXT: s_cselect_b64 s[6:7], -1, 0
; SDAG-NEXT: s_and_b64 s[6:7], s[6:7], vcc
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[6:7]
; SDAG-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[10:11]
; SDAG-NEXT: s_cbranch_execz .LBB0_7
; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-end9
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SDAG-NEXT: v_add_co_u32_e64 v9, s[4:5], -1, v0
; SDAG-NEXT: s_mov_b64 s[4:5], 0x432
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v9, vcc, -1, v0
; SDAG-NEXT: s_mov_b64 s[6:7], 0x432
; SDAG-NEXT: v_and_b32_e32 v0, 0xfffff, v5
; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[4:5], v[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v8, -1, 0, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v10, -1, 1, vcc
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v8, -1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v10, -1, 1, s[4:5]
; SDAG-NEXT: v_or_b32_e32 v5, 0x100000, v0
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[6:7]
; SDAG-NEXT: s_cbranch_execz .LBB0_4
; SDAG-NEXT: ; %bb.3: ; %fp-to-i-if-else
@ -55,35 +56,35 @@ define i128 @fptosi_f64_to_i128(double %x) {
; SDAG-NEXT: v_add_u32_e32 v3, 0xfffffbcd, v6
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, v[4:5]
; SDAG-NEXT: v_lshlrev_b64 v[6:7], v2, v[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v3
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
; SDAG-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v3
; SDAG-NEXT: v_lshlrev_b64 v[3:4], v3, v[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, v7, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v0, v6, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v3, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v3, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[6:7]
; SDAG-NEXT: v_mul_lo_u32 v12, v10, v1
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v7, v10, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 0
; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, v4, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v13, v10, v[1:2]
; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v4, vcc
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[6:7], v13, v10, v[1:2]
; SDAG-NEXT: v_mul_lo_u32 v11, v8, v5
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v10, v5, 0
; SDAG-NEXT: v_mov_b32_e32 v1, v3
; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v7, v8, v[1:2]
; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[6:7], v7, v8, v[1:2]
; SDAG-NEXT: v_add3_u32 v6, v6, v12, v11
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v7, v[5:6]
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v9, v7, v[5:6]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_mul_lo_u32 v10, v9, v13
; SDAG-NEXT: v_mul_lo_u32 v7, v9, v7
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v8, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v13, v8, v[2:3]
; SDAG-NEXT: ; implicit-def: $vgpr8
; SDAG-NEXT: ; implicit-def: $vgpr9
; SDAG-NEXT: v_add3_u32 v4, v7, v6, v10
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v2, v5
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], v3, v4, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7
; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; SDAG-NEXT: ; implicit-def: $vgpr10
@ -93,37 +94,37 @@ define i128 @fptosi_f64_to_i128(double %x) {
; SDAG-NEXT: ; %bb.5: ; %fp-to-i-if-then12
; SDAG-NEXT: v_sub_u32_e32 v2, 0x433, v6
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v2, v[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v2
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v2
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v6, v0, v4, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v5, v1, v5, s[6:7]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v10, 0
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v6, v10, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 0
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v10, v[1:2]
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[6:7], v5, v10, v[1:2]
; SDAG-NEXT: v_mov_b32_e32 v1, v3
; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v6, v8, v[1:2]
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v8, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v6, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v9, v6, v[3:4]
; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[6:7], v6, v8, v[1:2]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v5, v8, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v9, v6, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[6:7], v9, v6, v[3:4]
; SDAG-NEXT: v_mad_i32_i24 v3, v9, v5, v3
; SDAG-NEXT: .LBB0_6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB0_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
; SDAG-NEXT: s_andn2_saveexec_b64 s[6:7], s[10:11]
; SDAG-NEXT: ; %bb.8: ; %fp-to-i-if-then5
; SDAG-NEXT: v_bfrev_b32_e32 v0, 1
; SDAG-NEXT: v_bfrev_b32_e32 v1, -2
; SDAG-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v2, v0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
; SDAG-NEXT: v_mov_b32_e32 v3, v2
; SDAG-NEXT: v_mov_b32_e32 v0, v1
; SDAG-NEXT: v_mov_b32_e32 v2, v1
; SDAG-NEXT: ; %bb.9: ; %Flow3
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB0_10: ; %fp-to-i-cleanup
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_setpc_b64 s[30:31]
@ -386,32 +387,33 @@ define i128 @fptoui_f64_to_i128(double %x) {
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xfffffb81, v6
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: s_addc_u32 s6, 0, -1
; SDAG-NEXT: s_movk_i32 s10, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v7, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v7, vcc
; SDAG-NEXT: s_mov_b32 s7, -1
; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
; SDAG-NEXT: v_cmp_lt_i64_e32 vcc, -1, v[4:5]
; SDAG-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; SDAG-NEXT: s_addc_u32 s7, 0, -1
; SDAG-NEXT: s_mov_b32 s11, -1
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
; SDAG-NEXT: s_cmp_eq_u64 s[6:7], -1
; SDAG-NEXT: v_cmp_lt_i64_e64 s[4:5], -1, v[4:5]
; SDAG-NEXT: s_cselect_b64 s[6:7], -1, 0
; SDAG-NEXT: s_and_b64 s[6:7], s[6:7], vcc
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[6:7]
; SDAG-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[10:11]
; SDAG-NEXT: s_cbranch_execz .LBB1_7
; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-end9
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SDAG-NEXT: v_add_co_u32_e64 v9, s[4:5], -1, v0
; SDAG-NEXT: s_mov_b64 s[4:5], 0x432
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v9, vcc, -1, v0
; SDAG-NEXT: s_mov_b64 s[6:7], 0x432
; SDAG-NEXT: v_and_b32_e32 v0, 0xfffff, v5
; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[4:5], v[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v8, -1, 0, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v10, -1, 1, vcc
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v8, -1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v10, -1, 1, s[4:5]
; SDAG-NEXT: v_or_b32_e32 v5, 0x100000, v0
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[6:7]
; SDAG-NEXT: s_cbranch_execz .LBB1_4
; SDAG-NEXT: ; %bb.3: ; %fp-to-i-if-else
@ -420,35 +422,35 @@ define i128 @fptoui_f64_to_i128(double %x) {
; SDAG-NEXT: v_add_u32_e32 v3, 0xfffffbcd, v6
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, v[4:5]
; SDAG-NEXT: v_lshlrev_b64 v[6:7], v2, v[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v3
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v3
; SDAG-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v3
; SDAG-NEXT: v_lshlrev_b64 v[3:4], v3, v[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, v7, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v1, v7, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v0, v6, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v3, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v0, v6, v0, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v3, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v5, 0, v0, s[6:7]
; SDAG-NEXT: v_mul_lo_u32 v12, v10, v1
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v7, v10, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 0
; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, v4, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v13, v10, v[1:2]
; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v4, vcc
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[6:7], v13, v10, v[1:2]
; SDAG-NEXT: v_mul_lo_u32 v11, v8, v5
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v10, v5, 0
; SDAG-NEXT: v_mov_b32_e32 v1, v3
; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v7, v8, v[1:2]
; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[6:7], v7, v8, v[1:2]
; SDAG-NEXT: v_add3_u32 v6, v6, v12, v11
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v9, v7, v[5:6]
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v9, v7, v[5:6]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_mul_lo_u32 v10, v9, v13
; SDAG-NEXT: v_mul_lo_u32 v7, v9, v7
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v13, v8, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v13, v8, v[2:3]
; SDAG-NEXT: ; implicit-def: $vgpr8
; SDAG-NEXT: ; implicit-def: $vgpr9
; SDAG-NEXT: v_add3_u32 v4, v7, v6, v10
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v2, v5
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], v3, v4, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v2, v5
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v3, v4, vcc
; SDAG-NEXT: ; implicit-def: $vgpr6_vgpr7
; SDAG-NEXT: ; implicit-def: $vgpr4_vgpr5
; SDAG-NEXT: ; implicit-def: $vgpr10
@ -458,37 +460,37 @@ define i128 @fptoui_f64_to_i128(double %x) {
; SDAG-NEXT: ; %bb.5: ; %fp-to-i-if-then12
; SDAG-NEXT: v_sub_u32_e32 v2, 0x433, v6
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v2, v[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v2
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cmp_eq_u32_e64 s[6:7], 0, v2
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; SDAG-NEXT: v_cndmask_b32_e32 v1, 0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v6, v0, v4, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v5, v1, v5, s[6:7]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v6, v10, 0
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v6, v10, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 0
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v5, v10, v[1:2]
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[6:7], v5, v10, v[1:2]
; SDAG-NEXT: v_mov_b32_e32 v1, v3
; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[4:5], v6, v8, v[1:2]
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], 0, 0, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v5, v8, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v9, v6, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[4:5], v9, v6, v[3:4]
; SDAG-NEXT: v_mad_u64_u32 v[1:2], s[6:7], v6, v8, v[1:2]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v4, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v5, v8, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v9, v6, v[2:3]
; SDAG-NEXT: v_mad_u64_u32 v[3:4], s[6:7], v9, v6, v[3:4]
; SDAG-NEXT: v_mad_i32_i24 v3, v9, v5, v3
; SDAG-NEXT: .LBB1_6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[12:13]
; SDAG-NEXT: .LBB1_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
; SDAG-NEXT: s_andn2_saveexec_b64 s[6:7], s[10:11]
; SDAG-NEXT: ; %bb.8: ; %fp-to-i-if-then5
; SDAG-NEXT: v_bfrev_b32_e32 v0, 1
; SDAG-NEXT: v_bfrev_b32_e32 v1, -2
; SDAG-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v2, v0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
; SDAG-NEXT: v_mov_b32_e32 v3, v2
; SDAG-NEXT: v_mov_b32_e32 v0, v1
; SDAG-NEXT: v_mov_b32_e32 v2, v1
; SDAG-NEXT: ; %bb.9: ; %Flow3
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB1_10: ; %fp-to-i-cleanup
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_setpc_b64 s[30:31]
@ -749,34 +751,35 @@ define i128 @fptosi_f32_to_i128(float %x) {
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: s_addc_u32 s6, 0, -1
; SDAG-NEXT: s_movk_i32 s10, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
; SDAG-NEXT: s_mov_b32 s7, -1
; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, -1, v4
; SDAG-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; SDAG-NEXT: s_addc_u32 s7, 0, -1
; SDAG-NEXT: s_mov_b32 s11, -1
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
; SDAG-NEXT: s_cmp_eq_u64 s[6:7], -1
; SDAG-NEXT: s_cselect_b64 s[6:7], -1, 0
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_cmp_lt_i32_e64 s[4:5], -1, v4
; SDAG-NEXT: s_and_b64 s[6:7], s[6:7], vcc
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[6:7]
; SDAG-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[10:11]
; SDAG-NEXT: s_cbranch_execz .LBB2_7
; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-end9
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SDAG-NEXT: v_add_co_u32_e64 v10, s[4:5], -1, v0
; SDAG-NEXT: s_mov_b64 s[4:5], 0x95
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v10, vcc, -1, v0
; SDAG-NEXT: s_mov_b64 s[6:7], 0x95
; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v4
; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
; SDAG-NEXT: v_cndmask_b32_e64 v9, -1, 0, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v11, -1, 1, vcc
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
; SDAG-NEXT: v_cndmask_b32_e64 v9, -1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v11, -1, 1, s[4:5]
; SDAG-NEXT: v_or_b32_e32 v7, 0x800000, v0
; SDAG-NEXT: v_mov_b32_e32 v8, v6
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[6:7]
; SDAG-NEXT: s_cbranch_execz .LBB2_4
; SDAG-NEXT: ; %bb.3: ; %fp-to-i-if-else
@ -785,35 +788,35 @@ define i128 @fptosi_f32_to_i128(float %x) {
; SDAG-NEXT: v_add_u32_e32 v4, 0xffffff6a, v5
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, v[7:8]
; SDAG-NEXT: v_lshlrev_b64 v[2:3], v2, v[7:8]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v4
; SDAG-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
; SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; SDAG-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v4
; SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v1, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; SDAG-NEXT: v_lshlrev_b64 v[0:1], v4, v[7:8]
; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
; SDAG-NEXT: v_mad_u64_u32 v[4:5], s[6:7], v7, v11, 0
; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v1, vcc
; SDAG-NEXT: v_mul_lo_u32 v8, v9, v2
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v11, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v11, v[5:6]
; SDAG-NEXT: v_mul_lo_u32 v12, v11, v3
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v11, v2, 0
; SDAG-NEXT: v_mov_b32_e32 v5, v0
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v9, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v7, v9, v[5:6]
; SDAG-NEXT: v_add3_u32 v3, v3, v12, v8
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v7, v[2:3]
; SDAG-NEXT: v_add_co_u32_e64 v0, s[4:5], v1, v6
; SDAG-NEXT: v_addc_co_u32_e64 v1, s[4:5], 0, 0, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v10, v7, v[2:3]
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v1, v6
; SDAG-NEXT: v_addc_co_u32_e64 v1, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_mul_lo_u32 v8, v10, v13
; SDAG-NEXT: v_mul_lo_u32 v7, v10, v7
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v9, v[0:1]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v9, v[0:1]
; SDAG-NEXT: ; implicit-def: $vgpr11
; SDAG-NEXT: ; implicit-def: $vgpr9
; SDAG-NEXT: ; implicit-def: $vgpr10
; SDAG-NEXT: v_add3_u32 v3, v7, v3, v8
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v0, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], v1, v3, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; SDAG-NEXT: v_mov_b32_e32 v0, v4
; SDAG-NEXT: v_mov_b32_e32 v1, v5
; SDAG-NEXT: ; implicit-def: $vgpr5_vgpr6
@ -824,30 +827,30 @@ define i128 @fptosi_f32_to_i128(float %x) {
; SDAG-NEXT: ; %bb.5: ; %fp-to-i-if-then12
; SDAG-NEXT: v_sub_u32_e32 v2, 0x96, v5
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v2, v[7:8]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v2
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
; SDAG-NEXT: v_cndmask_b32_e64 v3, v0, v7, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v11, 0
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; SDAG-NEXT: v_cndmask_b32_e32 v3, v0, v7, vcc
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[12:13], v3, v11, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 0
; SDAG-NEXT: v_mov_b32_e32 v6, v2
; SDAG-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, v9, v[1:2]
; SDAG-NEXT: v_mad_i64_i32 v[2:3], s[4:5], v10, v3, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[4:5], s[12:13], v3, v9, v[1:2]
; SDAG-NEXT: v_mad_i64_i32 v[2:3], s[12:13], v10, v3, v[5:6]
; SDAG-NEXT: v_mov_b32_e32 v1, v4
; SDAG-NEXT: .LBB2_6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB2_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
; SDAG-NEXT: s_andn2_saveexec_b64 s[6:7], s[10:11]
; SDAG-NEXT: ; %bb.8: ; %fp-to-i-if-then5
; SDAG-NEXT: v_bfrev_b32_e32 v0, 1
; SDAG-NEXT: v_bfrev_b32_e32 v1, -2
; SDAG-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v2, v0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
; SDAG-NEXT: v_mov_b32_e32 v3, v2
; SDAG-NEXT: v_mov_b32_e32 v0, v1
; SDAG-NEXT: v_mov_b32_e32 v2, v1
; SDAG-NEXT: ; %bb.9: ; %Flow3
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB2_10: ; %fp-to-i-cleanup
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_setpc_b64 s[30:31]
@ -1103,34 +1106,35 @@ define i128 @fptoui_f32_to_i128(float %x) {
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: s_addc_u32 s6, 0, -1
; SDAG-NEXT: s_movk_i32 s10, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
; SDAG-NEXT: s_mov_b32 s7, -1
; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
; SDAG-NEXT: v_cmp_lt_i32_e32 vcc, -1, v4
; SDAG-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; SDAG-NEXT: s_addc_u32 s7, 0, -1
; SDAG-NEXT: s_mov_b32 s11, -1
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
; SDAG-NEXT: s_cmp_eq_u64 s[6:7], -1
; SDAG-NEXT: s_cselect_b64 s[6:7], -1, 0
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_cmp_lt_i32_e64 s[4:5], -1, v4
; SDAG-NEXT: s_and_b64 s[6:7], s[6:7], vcc
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[6:7]
; SDAG-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[10:11]
; SDAG-NEXT: s_cbranch_execz .LBB3_7
; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-end9
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SDAG-NEXT: v_add_co_u32_e64 v10, s[4:5], -1, v0
; SDAG-NEXT: s_mov_b64 s[4:5], 0x95
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v10, vcc, -1, v0
; SDAG-NEXT: s_mov_b64 s[6:7], 0x95
; SDAG-NEXT: v_and_b32_e32 v0, 0x7fffff, v4
; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
; SDAG-NEXT: v_cndmask_b32_e64 v9, -1, 0, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v11, -1, 1, vcc
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
; SDAG-NEXT: v_cndmask_b32_e64 v9, -1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v11, -1, 1, s[4:5]
; SDAG-NEXT: v_or_b32_e32 v7, 0x800000, v0
; SDAG-NEXT: v_mov_b32_e32 v8, v6
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[6:7]
; SDAG-NEXT: s_cbranch_execz .LBB3_4
; SDAG-NEXT: ; %bb.3: ; %fp-to-i-if-else
@ -1139,35 +1143,35 @@ define i128 @fptoui_f32_to_i128(float %x) {
; SDAG-NEXT: v_add_u32_e32 v4, 0xffffff6a, v5
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, v[7:8]
; SDAG-NEXT: v_lshlrev_b64 v[2:3], v2, v[7:8]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v4
; SDAG-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
; SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; SDAG-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v4
; SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v1, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; SDAG-NEXT: v_lshlrev_b64 v[0:1], v4, v[7:8]
; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
; SDAG-NEXT: v_mad_u64_u32 v[4:5], s[6:7], v7, v11, 0
; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v1, vcc
; SDAG-NEXT: v_mul_lo_u32 v8, v9, v2
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v11, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v11, v[5:6]
; SDAG-NEXT: v_mul_lo_u32 v12, v11, v3
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v11, v2, 0
; SDAG-NEXT: v_mov_b32_e32 v5, v0
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v9, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v7, v9, v[5:6]
; SDAG-NEXT: v_add3_u32 v3, v3, v12, v8
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v10, v7, v[2:3]
; SDAG-NEXT: v_add_co_u32_e64 v0, s[4:5], v1, v6
; SDAG-NEXT: v_addc_co_u32_e64 v1, s[4:5], 0, 0, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v10, v7, v[2:3]
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v1, v6
; SDAG-NEXT: v_addc_co_u32_e64 v1, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_mul_lo_u32 v8, v10, v13
; SDAG-NEXT: v_mul_lo_u32 v7, v10, v7
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v9, v[0:1]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v9, v[0:1]
; SDAG-NEXT: ; implicit-def: $vgpr11
; SDAG-NEXT: ; implicit-def: $vgpr9
; SDAG-NEXT: ; implicit-def: $vgpr10
; SDAG-NEXT: v_add3_u32 v3, v7, v3, v8
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v0, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], v1, v3, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; SDAG-NEXT: v_mov_b32_e32 v0, v4
; SDAG-NEXT: v_mov_b32_e32 v1, v5
; SDAG-NEXT: ; implicit-def: $vgpr5_vgpr6
@ -1178,30 +1182,30 @@ define i128 @fptoui_f32_to_i128(float %x) {
; SDAG-NEXT: ; %bb.5: ; %fp-to-i-if-then12
; SDAG-NEXT: v_sub_u32_e32 v2, 0x96, v5
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v2, v[7:8]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v2
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
; SDAG-NEXT: v_cndmask_b32_e64 v3, v0, v7, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v3, v11, 0
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; SDAG-NEXT: v_cndmask_b32_e32 v3, v0, v7, vcc
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[12:13], v3, v11, 0
; SDAG-NEXT: v_mov_b32_e32 v2, 0
; SDAG-NEXT: v_mov_b32_e32 v6, v2
; SDAG-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v3, v9, v[1:2]
; SDAG-NEXT: v_mad_i64_i32 v[2:3], s[4:5], v10, v3, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[4:5], s[12:13], v3, v9, v[1:2]
; SDAG-NEXT: v_mad_i64_i32 v[2:3], s[12:13], v10, v3, v[5:6]
; SDAG-NEXT: v_mov_b32_e32 v1, v4
; SDAG-NEXT: .LBB3_6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB3_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
; SDAG-NEXT: s_andn2_saveexec_b64 s[6:7], s[10:11]
; SDAG-NEXT: ; %bb.8: ; %fp-to-i-if-then5
; SDAG-NEXT: v_bfrev_b32_e32 v0, 1
; SDAG-NEXT: v_bfrev_b32_e32 v1, -2
; SDAG-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v2, v0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
; SDAG-NEXT: v_mov_b32_e32 v3, v2
; SDAG-NEXT: v_mov_b32_e32 v0, v1
; SDAG-NEXT: v_mov_b32_e32 v2, v1
; SDAG-NEXT: ; %bb.9: ; %Flow3
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB3_10: ; %fp-to-i-cleanup
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_setpc_b64 s[30:31]
@ -1495,70 +1499,71 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: s_addc_u32 s6, 0, -1
; SDAG-NEXT: s_movk_i32 s10, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
; SDAG-NEXT: s_mov_b32 s7, -1
; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
; SDAG-NEXT: v_cmp_lt_i16_e32 vcc, -1, v4
; SDAG-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; SDAG-NEXT: s_addc_u32 s7, 0, -1
; SDAG-NEXT: s_mov_b32 s11, -1
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
; SDAG-NEXT: s_cmp_eq_u64 s[6:7], -1
; SDAG-NEXT: s_cselect_b64 s[6:7], -1, 0
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_cmp_lt_i16_e64 s[4:5], -1, v4
; SDAG-NEXT: s_and_b64 s[6:7], s[6:7], vcc
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[6:7]
; SDAG-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[10:11]
; SDAG-NEXT: s_cbranch_execz .LBB6_7
; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-end9
; SDAG-NEXT: s_movk_i32 s4, 0x7f
; SDAG-NEXT: v_and_b32_sdwa v0, v4, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; SDAG-NEXT: s_mov_b64 s[4:5], 0x85
; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
; SDAG-NEXT: v_cndmask_b32_e64 v10, -1, 0, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v9, -1, 1, vcc
; SDAG-NEXT: s_movk_i32 s6, 0x7f
; SDAG-NEXT: v_and_b32_sdwa v0, v4, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; SDAG-NEXT: s_mov_b64 s[6:7], 0x85
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
; SDAG-NEXT: v_cndmask_b32_e64 v10, -1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v9, -1, 1, s[4:5]
; SDAG-NEXT: v_or_b32_e32 v7, 0x80, v0
; SDAG-NEXT: v_mov_b32_e32 v8, v6
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[6:7]
; SDAG-NEXT: s_cbranch_execz .LBB6_4
; SDAG-NEXT: ; %bb.3: ; %fp-to-i-if-else
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SDAG-NEXT: v_add_co_u32_e64 v11, s[4:5], -1, v0
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v11, vcc, -1, v0
; SDAG-NEXT: v_sub_u32_e32 v0, 0xc6, v5
; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff3a, v5
; SDAG-NEXT: v_add_u32_e32 v4, 0xffffff7a, v5
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, v[7:8]
; SDAG-NEXT: v_lshlrev_b64 v[2:3], v2, v[7:8]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v4
; SDAG-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
; SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; SDAG-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v4
; SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v1, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; SDAG-NEXT: v_lshlrev_b64 v[0:1], v4, v[7:8]
; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
; SDAG-NEXT: v_mad_u64_u32 v[4:5], s[6:7], v7, v9, 0
; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v1, vcc
; SDAG-NEXT: v_mul_lo_u32 v8, v10, v2
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v9, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v9, v[5:6]
; SDAG-NEXT: v_mul_lo_u32 v12, v9, v3
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v9, v2, 0
; SDAG-NEXT: v_mov_b32_e32 v5, v0
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v10, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v7, v10, v[5:6]
; SDAG-NEXT: v_add3_u32 v3, v3, v12, v8
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v7, v[2:3]
; SDAG-NEXT: v_add_co_u32_e64 v0, s[4:5], v1, v6
; SDAG-NEXT: v_addc_co_u32_e64 v1, s[4:5], 0, 0, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v11, v7, v[2:3]
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v1, v6
; SDAG-NEXT: v_addc_co_u32_e64 v1, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_mul_lo_u32 v8, v11, v13
; SDAG-NEXT: v_mul_lo_u32 v7, v11, v7
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v10, v[0:1]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v10, v[0:1]
; SDAG-NEXT: ; implicit-def: $vgpr9
; SDAG-NEXT: v_add3_u32 v3, v7, v3, v8
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v0, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], v1, v3, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; SDAG-NEXT: v_mov_b32_e32 v0, v4
; SDAG-NEXT: v_mov_b32_e32 v1, v5
; SDAG-NEXT: ; implicit-def: $vgpr5_vgpr6
@ -1569,10 +1574,10 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
; SDAG-NEXT: ; %bb.5: ; %fp-to-i-if-then12
; SDAG-NEXT: v_sub_u32_e32 v2, 0x86, v5
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v2, v[7:8]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v2
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v7, s[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; SDAG-NEXT: v_mul_hi_i32_i24_e32 v1, v0, v9
; SDAG-NEXT: v_ashrrev_i32_e32 v2, 31, v1
; SDAG-NEXT: v_mul_i32_i24_e32 v0, v0, v9
@ -1580,17 +1585,17 @@ define i128 @fptosi_bf16_to_i128(bfloat %x) {
; SDAG-NEXT: .LBB6_6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB6_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
; SDAG-NEXT: s_andn2_saveexec_b64 s[6:7], s[10:11]
; SDAG-NEXT: ; %bb.8: ; %fp-to-i-if-then5
; SDAG-NEXT: v_bfrev_b32_e32 v0, 1
; SDAG-NEXT: v_bfrev_b32_e32 v1, -2
; SDAG-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v2, v0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
; SDAG-NEXT: v_mov_b32_e32 v3, v2
; SDAG-NEXT: v_mov_b32_e32 v0, v1
; SDAG-NEXT: v_mov_b32_e32 v2, v1
; SDAG-NEXT: ; %bb.9: ; %Flow3
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB6_10: ; %fp-to-i-cleanup
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_setpc_b64 s[30:31]
@ -1843,70 +1848,71 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
; SDAG-NEXT: ; %bb.1: ; %fp-to-i-if-end
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, 0xffffff01, v5
; SDAG-NEXT: v_mov_b32_e32 v1, -1
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: s_addc_u32 s6, 0, -1
; SDAG-NEXT: s_movk_i32 s10, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v1, vcc, 0, v1, vcc
; SDAG-NEXT: v_addc_co_u32_e32 v2, vcc, -1, v6, vcc
; SDAG-NEXT: s_movk_i32 s6, 0xff7f
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, -1, v6, vcc
; SDAG-NEXT: s_mov_b32 s7, -1
; SDAG-NEXT: v_cmp_eq_u64_e64 s[4:5], -1, v[2:3]
; SDAG-NEXT: v_cmp_lt_u64_e64 s[6:7], s[6:7], v[0:1]
; SDAG-NEXT: v_cmp_lt_i16_e32 vcc, -1, v4
; SDAG-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; SDAG-NEXT: s_addc_u32 s7, 0, -1
; SDAG-NEXT: s_mov_b32 s11, -1
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[10:11], v[0:1]
; SDAG-NEXT: s_cmp_eq_u64 s[6:7], -1
; SDAG-NEXT: s_cselect_b64 s[6:7], -1, 0
; SDAG-NEXT: v_mov_b32_e32 v6, 0
; SDAG-NEXT: v_cmp_lt_i16_e64 s[4:5], -1, v4
; SDAG-NEXT: s_and_b64 s[6:7], s[6:7], vcc
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[6:7]
; SDAG-NEXT: s_and_saveexec_b64 s[10:11], s[6:7]
; SDAG-NEXT: s_xor_b64 s[10:11], exec, s[10:11]
; SDAG-NEXT: s_cbranch_execz .LBB7_7
; SDAG-NEXT: ; %bb.2: ; %fp-to-i-if-end9
; SDAG-NEXT: s_movk_i32 s4, 0x7f
; SDAG-NEXT: v_and_b32_sdwa v0, v4, s4 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; SDAG-NEXT: s_mov_b64 s[4:5], 0x85
; SDAG-NEXT: v_cmp_lt_u64_e64 s[4:5], s[4:5], v[5:6]
; SDAG-NEXT: v_cndmask_b32_e64 v10, -1, 0, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v9, -1, 1, vcc
; SDAG-NEXT: s_movk_i32 s6, 0x7f
; SDAG-NEXT: v_and_b32_sdwa v0, v4, s6 dst_sel:DWORD dst_unused:UNUSED_PAD src0_sel:WORD_0 src1_sel:DWORD
; SDAG-NEXT: s_mov_b64 s[6:7], 0x85
; SDAG-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[5:6]
; SDAG-NEXT: v_cndmask_b32_e64 v10, -1, 0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v9, -1, 1, s[4:5]
; SDAG-NEXT: v_or_b32_e32 v7, 0x80, v0
; SDAG-NEXT: v_mov_b32_e32 v8, v6
; SDAG-NEXT: ; implicit-def: $vgpr0_vgpr1
; SDAG-NEXT: ; implicit-def: $vgpr2_vgpr3
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], s[4:5]
; SDAG-NEXT: s_and_saveexec_b64 s[6:7], vcc
; SDAG-NEXT: s_xor_b64 s[12:13], exec, s[6:7]
; SDAG-NEXT: s_cbranch_execz .LBB7_4
; SDAG-NEXT: ; %bb.3: ; %fp-to-i-if-else
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, vcc
; SDAG-NEXT: v_add_co_u32_e64 v11, s[4:5], -1, v0
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, 1, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v11, vcc, -1, v0
; SDAG-NEXT: v_sub_u32_e32 v0, 0xc6, v5
; SDAG-NEXT: v_add_u32_e32 v2, 0xffffff3a, v5
; SDAG-NEXT: v_add_u32_e32 v4, 0xffffff7a, v5
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v0, v[7:8]
; SDAG-NEXT: v_lshlrev_b64 v[2:3], v2, v[7:8]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v4
; SDAG-NEXT: v_cndmask_b32_e64 v1, v3, v1, s[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v4
; SDAG-NEXT: v_cndmask_b32_e32 v1, v3, v1, vcc
; SDAG-NEXT: v_cmp_ne_u32_e64 s[6:7], 0, v4
; SDAG-NEXT: v_cndmask_b32_e64 v3, 0, v1, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v2, v2, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v2, v2, v0, vcc
; SDAG-NEXT: v_lshlrev_b64 v[0:1], v4, v[7:8]
; SDAG-NEXT: v_cndmask_b32_e64 v2, 0, v2, s[6:7]
; SDAG-NEXT: v_cndmask_b32_e64 v7, 0, v0, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v7, 0, v0, vcc
; SDAG-NEXT: v_mad_u64_u32 v[4:5], s[6:7], v7, v9, 0
; SDAG-NEXT: v_cndmask_b32_e64 v13, 0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e32 v13, 0, v1, vcc
; SDAG-NEXT: v_mul_lo_u32 v8, v10, v2
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v9, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v9, v[5:6]
; SDAG-NEXT: v_mul_lo_u32 v12, v9, v3
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v9, v2, 0
; SDAG-NEXT: v_mov_b32_e32 v5, v0
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v7, v10, v[5:6]
; SDAG-NEXT: v_mad_u64_u32 v[5:6], s[6:7], v7, v10, v[5:6]
; SDAG-NEXT: v_add3_u32 v3, v3, v12, v8
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[4:5], v11, v7, v[2:3]
; SDAG-NEXT: v_add_co_u32_e64 v0, s[4:5], v1, v6
; SDAG-NEXT: v_addc_co_u32_e64 v1, s[4:5], 0, 0, s[4:5]
; SDAG-NEXT: v_mad_u64_u32 v[2:3], s[6:7], v11, v7, v[2:3]
; SDAG-NEXT: v_add_co_u32_e32 v0, vcc, v1, v6
; SDAG-NEXT: v_addc_co_u32_e64 v1, s[6:7], 0, 0, vcc
; SDAG-NEXT: v_mul_lo_u32 v8, v11, v13
; SDAG-NEXT: v_mul_lo_u32 v7, v11, v7
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[4:5], v13, v10, v[0:1]
; SDAG-NEXT: v_mad_u64_u32 v[0:1], s[6:7], v13, v10, v[0:1]
; SDAG-NEXT: ; implicit-def: $vgpr9
; SDAG-NEXT: v_add3_u32 v3, v7, v3, v8
; SDAG-NEXT: v_add_co_u32_e64 v2, s[4:5], v0, v2
; SDAG-NEXT: v_addc_co_u32_e64 v3, s[4:5], v1, v3, s[4:5]
; SDAG-NEXT: v_add_co_u32_e32 v2, vcc, v0, v2
; SDAG-NEXT: v_addc_co_u32_e32 v3, vcc, v1, v3, vcc
; SDAG-NEXT: v_mov_b32_e32 v0, v4
; SDAG-NEXT: v_mov_b32_e32 v1, v5
; SDAG-NEXT: ; implicit-def: $vgpr5_vgpr6
@ -1917,10 +1923,10 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
; SDAG-NEXT: ; %bb.5: ; %fp-to-i-if-then12
; SDAG-NEXT: v_sub_u32_e32 v2, 0x86, v5
; SDAG-NEXT: v_lshrrev_b64 v[0:1], v2, v[7:8]
; SDAG-NEXT: v_cmp_gt_u32_e64 s[4:5], 64, v2
; SDAG-NEXT: v_cndmask_b32_e64 v0, 0, v0, s[4:5]
; SDAG-NEXT: v_cmp_eq_u32_e64 s[4:5], 0, v2
; SDAG-NEXT: v_cndmask_b32_e64 v0, v0, v7, s[4:5]
; SDAG-NEXT: v_cmp_gt_u32_e32 vcc, 64, v2
; SDAG-NEXT: v_cndmask_b32_e32 v0, 0, v0, vcc
; SDAG-NEXT: v_cmp_eq_u32_e32 vcc, 0, v2
; SDAG-NEXT: v_cndmask_b32_e32 v0, v0, v7, vcc
; SDAG-NEXT: v_mul_hi_i32_i24_e32 v1, v0, v9
; SDAG-NEXT: v_ashrrev_i32_e32 v2, 31, v1
; SDAG-NEXT: v_mul_i32_i24_e32 v0, v0, v9
@ -1928,17 +1934,17 @@ define i128 @fptoui_bf16_to_i128(bfloat %x) {
; SDAG-NEXT: .LBB7_6: ; %Flow1
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB7_7: ; %Flow2
; SDAG-NEXT: s_andn2_saveexec_b64 s[4:5], s[10:11]
; SDAG-NEXT: s_andn2_saveexec_b64 s[6:7], s[10:11]
; SDAG-NEXT: ; %bb.8: ; %fp-to-i-if-then5
; SDAG-NEXT: v_bfrev_b32_e32 v0, 1
; SDAG-NEXT: v_bfrev_b32_e32 v1, -2
; SDAG-NEXT: v_cndmask_b32_e32 v2, v0, v1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, vcc
; SDAG-NEXT: v_cndmask_b32_e64 v2, v0, v1, s[4:5]
; SDAG-NEXT: v_cndmask_b32_e64 v1, 0, -1, s[4:5]
; SDAG-NEXT: v_mov_b32_e32 v3, v2
; SDAG-NEXT: v_mov_b32_e32 v0, v1
; SDAG-NEXT: v_mov_b32_e32 v2, v1
; SDAG-NEXT: ; %bb.9: ; %Flow3
; SDAG-NEXT: s_or_b64 exec, exec, s[4:5]
; SDAG-NEXT: s_or_b64 exec, exec, s[6:7]
; SDAG-NEXT: .LBB7_10: ; %fp-to-i-cleanup
; SDAG-NEXT: s_or_b64 exec, exec, s[8:9]
; SDAG-NEXT: s_setpc_b64 s[30:31]

View File

@ -25,12 +25,12 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, 0, v6, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v7, vcc
; GFX9-NEXT: v_cmp_gt_i64_e32 vcc, 0, v[6:7]
; GFX9-NEXT: v_mov_b32_e32 v21, v20
; GFX9-NEXT: v_cndmask_b32_e32 v22, v5, v9, vcc
; GFX9-NEXT: s_mov_b64 s[8:9], 0x7f
; GFX9-NEXT: v_cndmask_b32_e32 v21, v5, v9, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v23, v4, v8, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v5, v7, v11, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v4, v6, v10, vcc
; GFX9-NEXT: v_or_b32_e32 v7, v22, v5
; GFX9-NEXT: v_or_b32_e32 v7, v21, v5
; GFX9-NEXT: v_or_b32_e32 v6, v23, v4
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[6:7]
; GFX9-NEXT: v_or_b32_e32 v7, v1, v3
@ -42,56 +42,59 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_min_u32_e32 v6, v6, v7
; GFX9-NEXT: v_ffbh_u32_e32 v7, v23
; GFX9-NEXT: v_add_u32_e32 v7, 32, v7
; GFX9-NEXT: v_ffbh_u32_e32 v8, v22
; GFX9-NEXT: v_ffbh_u32_e32 v8, v21
; GFX9-NEXT: v_min_u32_e32 v7, v7, v8
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, 64, v7
; GFX9-NEXT: v_addc_co_u32_e64 v8, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v8, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[4:5]
; GFX9-NEXT: v_ffbh_u32_e32 v10, v3
; GFX9-NEXT: v_ffbh_u32_e32 v9, v3
; GFX9-NEXT: v_cndmask_b32_e32 v6, v7, v6, vcc
; GFX9-NEXT: v_ffbh_u32_e32 v7, v2
; GFX9-NEXT: v_add_u32_e32 v7, 32, v7
; GFX9-NEXT: v_min_u32_e32 v7, v7, v10
; GFX9-NEXT: v_ffbh_u32_e32 v10, v0
; GFX9-NEXT: v_add_u32_e32 v10, 32, v10
; GFX9-NEXT: v_ffbh_u32_e32 v11, v1
; GFX9-NEXT: v_min_u32_e32 v10, v10, v11
; GFX9-NEXT: v_min_u32_e32 v7, v7, v9
; GFX9-NEXT: v_ffbh_u32_e32 v9, v0
; GFX9-NEXT: v_add_u32_e32 v9, 32, v9
; GFX9-NEXT: v_ffbh_u32_e32 v10, v1
; GFX9-NEXT: v_min_u32_e32 v9, v9, v10
; GFX9-NEXT: v_cndmask_b32_e64 v8, v8, 0, vcc
; GFX9-NEXT: v_add_co_u32_e32 v10, vcc, 64, v10
; GFX9-NEXT: v_addc_co_u32_e64 v11, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9
; GFX9-NEXT: v_addc_co_u32_e64 v10, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX9-NEXT: v_mov_b32_e32 v9, 0
; GFX9-NEXT: v_cndmask_b32_e32 v7, v10, v7, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v11, v11, 0, vcc
; GFX9-NEXT: s_subb_u32 s4, 0, 0
; GFX9-NEXT: v_cndmask_b32_e32 v7, v9, v7, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v6, vcc, v6, v7
; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v8, v11, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v8, vcc, 0, v9, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, 0, v9, vcc
; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v7, vcc, v8, v10, vcc
; GFX9-NEXT: s_subb_u32 s5, 0, 0
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[6:7]
; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-NEXT: s_cmp_eq_u64 s[4:5], 0
; GFX9-NEXT: v_cndmask_b32_e64 v8, 0, 1, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[8:9]
; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
; GFX9-NEXT: v_and_b32_e32 v8, 1, v8
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v8
; GFX9-NEXT: v_xor_b32_e32 v8, 0x7f, v6
; GFX9-NEXT: v_or_b32_e32 v9, s5, v7
; GFX9-NEXT: v_or_b32_e32 v8, s4, v8
; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[8:9]
; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, 1, vcc
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[8:9]
; GFX9-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
; GFX9-NEXT: v_and_b32_e32 v10, 1, v10
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
; GFX9-NEXT: v_xor_b32_e32 v10, 0x7f, v6
; GFX9-NEXT: v_or_b32_e32 v11, v7, v9
; GFX9-NEXT: v_or_b32_e32 v10, v10, v8
; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1
; GFX9-NEXT: v_cndmask_b32_e64 v13, v3, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v12, v2, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v11, v1, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v10, v0, 0, s[4:5]
; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
; GFX9-NEXT: s_xor_b64 s[8:9], s[6:7], -1
; GFX9-NEXT: v_mov_b32_e32 v22, v20
; GFX9-NEXT: v_cndmask_b32_e64 v13, v3, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v12, v2, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v8, v1, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v9, v0, 0, s[6:7]
; GFX9-NEXT: s_and_b64 s[6:7], s[8:9], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[6:7]
; GFX9-NEXT: s_cbranch_execz .LBB0_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
; GFX9-NEXT: v_mov_b32_e32 v9, s5
; GFX9-NEXT: v_add_co_u32_e32 v24, vcc, 1, v6
; GFX9-NEXT: v_mov_b32_e32 v8, s4
; GFX9-NEXT: v_addc_co_u32_e32 v25, vcc, 0, v7, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v26, vcc, 0, v8, vcc
; GFX9-NEXT: v_sub_u32_e32 v13, 0x7f, v6
@ -138,7 +141,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_cndmask_b32_e32 v17, 0, v11, vcc
; GFX9-NEXT: v_cndmask_b32_e32 v16, 0, v10, vcc
; GFX9-NEXT: v_add_co_u32_e32 v28, vcc, -1, v23
; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v22, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v29, vcc, -1, v21, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v30, vcc, -1, v4, vcc
; GFX9-NEXT: v_mov_b32_e32 v12, 0
; GFX9-NEXT: v_mov_b32_e32 v18, 0
@ -167,7 +170,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_or_b32_e32 v8, v18, v8
; GFX9-NEXT: v_and_b32_e32 v18, v10, v23
; GFX9-NEXT: v_or_b32_e32 v9, v19, v9
; GFX9-NEXT: v_and_b32_e32 v19, v10, v22
; GFX9-NEXT: v_and_b32_e32 v19, v10, v21
; GFX9-NEXT: v_sub_co_u32_e32 v14, vcc, v14, v18
; GFX9-NEXT: v_and_b32_e32 v32, v10, v4
; GFX9-NEXT: v_subb_co_u32_e32 v15, vcc, v15, v19, vcc
@ -196,40 +199,40 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_lshlrev_b64 v[6:7], 1, v[8:9]
; GFX9-NEXT: v_lshrrev_b32_e32 v14, 31, v9
; GFX9-NEXT: v_or_b32_e32 v12, v12, v14
; GFX9-NEXT: v_or_b32_e32 v11, v11, v7
; GFX9-NEXT: v_or_b32_e32 v10, v10, v6
; GFX9-NEXT: v_or_b32_e32 v8, v11, v7
; GFX9-NEXT: v_or_b32_e32 v9, v10, v6
; GFX9-NEXT: .LBB0_6: ; %Flow3
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: v_mul_lo_u32 v17, v10, v5
; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v23, v10, 0
; GFX9-NEXT: v_mul_lo_u32 v17, v9, v5
; GFX9-NEXT: v_mad_u64_u32 v[5:6], s[4:5], v23, v9, 0
; GFX9-NEXT: v_mov_b32_e32 v7, 0
; GFX9-NEXT: v_mul_lo_u32 v16, v11, v4
; GFX9-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v22, v10, v[6:7]
; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v10, v4, 0
; GFX9-NEXT: v_mul_lo_u32 v16, v8, v4
; GFX9-NEXT: v_mad_u64_u32 v[14:15], s[4:5], v21, v9, v[6:7]
; GFX9-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v9, v4, 0
; GFX9-NEXT: v_mov_b32_e32 v6, v14
; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v23, v11, v[6:7]
; GFX9-NEXT: v_add3_u32 v9, v9, v17, v16
; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v12, v23, v[8:9]
; GFX9-NEXT: v_mul_lo_u32 v4, v12, v22
; GFX9-NEXT: v_add_co_u32_e32 v12, vcc, v15, v7
; GFX9-NEXT: v_mul_lo_u32 v14, v13, v23
; GFX9-NEXT: v_addc_co_u32_e64 v13, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v22, v11, v[12:13]
; GFX9-NEXT: v_add3_u32 v4, v14, v9, v4
; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v10, v8
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v11, v4, vcc
; GFX9-NEXT: v_mad_u64_u32 v[6:7], s[4:5], v23, v8, v[6:7]
; GFX9-NEXT: v_add3_u32 v11, v11, v17, v16
; GFX9-NEXT: v_mad_u64_u32 v[9:10], s[4:5], v12, v23, v[10:11]
; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, v15, v7
; GFX9-NEXT: v_mul_lo_u32 v4, v12, v21
; GFX9-NEXT: v_addc_co_u32_e64 v12, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_mul_lo_u32 v13, v13, v23
; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v21, v8, v[11:12]
; GFX9-NEXT: v_add3_u32 v4, v13, v10, v4
; GFX9-NEXT: v_add_co_u32_e32 v7, vcc, v7, v9
; GFX9-NEXT: v_addc_co_u32_e32 v4, vcc, v8, v4, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v5
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v6, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v7, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v4, vcc
; GFX9-NEXT: v_xor_b32_e32 v0, v0, v20
; GFX9-NEXT: v_xor_b32_e32 v1, v1, v21
; GFX9-NEXT: v_xor_b32_e32 v1, v1, v22
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v20
; GFX9-NEXT: v_xor_b32_e32 v2, v2, v20
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v21, vcc
; GFX9-NEXT: v_xor_b32_e32 v3, v3, v21
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v22, vcc
; GFX9-NEXT: v_xor_b32_e32 v3, v3, v22
; GFX9-NEXT: v_subb_co_u32_e32 v2, vcc, v2, v20, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v21, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v3, vcc, v3, v22, vcc
; GFX9-NEXT: s_setpc_b64 s[30:31]
;
; GFX9-O0-LABEL: v_srem_i128_vv:
@ -269,9 +272,7 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 0
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 1
; GFX9-O0-NEXT: s_mov_b32 s10, s6
; GFX9-O0-NEXT: v_writelane_b32 v30, s10, 2
; GFX9-O0-NEXT: s_mov_b32 s11, s7
; GFX9-O0-NEXT: v_writelane_b32 v30, s11, 3
; GFX9-O0-NEXT: v_sub_co_u32_e32 v10, vcc, s10, v2
; GFX9-O0-NEXT: v_mov_b32_e32 v4, s11
; GFX9-O0-NEXT: v_subb_co_u32_e32 v5, vcc, v4, v3, vcc
@ -453,49 +454,49 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: ; kill: def $vgpr6 killed $vgpr6 killed $vgpr5_vgpr6 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v9
; GFX9-O0-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v7
; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v5, v6, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s10
; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v5, v6, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s11
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11
; GFX9-O0-NEXT: v_subb_co_u32_e32 v6, vcc, v5, v6, vcc
; GFX9-O0-NEXT: s_subb_u32 s12, s10, s10
; GFX9-O0-NEXT: s_subb_u32 s8, s11, s11
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 def $sgpr12_sgpr13
; GFX9-O0-NEXT: s_mov_b32 s13, s8
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v8
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s12
; GFX9-O0-NEXT: v_mov_b32_e32 v7, s13
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
; GFX9-O0-NEXT: s_mov_b64 s[12:13], 0x7f
; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[14:15]
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[14:15]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[8:9]
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[14:15], 0x7f
; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[8:9], v[4:5], s[14:15]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[8:9]
; GFX9-O0-NEXT: s_cmp_lg_u64 s[12:13], s[6:7]
; GFX9-O0-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[8:9]
; GFX9-O0-NEXT: s_cmp_eq_u64 s[12:13], s[6:7]
; GFX9-O0-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9]
; GFX9-O0-NEXT: v_and_b32_e64 v6, 1, v6
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[8:9], v6, 1
; GFX9-O0-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GFX9-O0-NEXT: s_mov_b64 s[14:15], -1
; GFX9-O0-NEXT: s_mov_b64 s[16:17], -1
; GFX9-O0-NEXT: s_mov_b64 s[4:5], s[8:9]
; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[4:5], s[14:15]
; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[4:5], s[16:17]
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5
; GFX9-O0-NEXT: s_mov_b32 s14, s13
; GFX9-O0-NEXT: v_xor_b32_e64 v6, v6, s14
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, s12
; GFX9-O0-NEXT: s_mov_b32 s16, s15
; GFX9-O0-NEXT: v_xor_b32_e64 v6, v6, s16
; GFX9-O0-NEXT: ; kill: def $sgpr14 killed $sgpr14 killed $sgpr14_sgpr15
; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, s14
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v8
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v9
; GFX9-O0-NEXT: s_mov_b32 s14, s13
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, s14
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v5
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, s12
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[4:5], s[6:7]
@ -522,8 +523,8 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 4
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 5
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 2
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 3
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
@ -536,8 +537,8 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 4
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 5
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: ; %bb.2: ; %Flow
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:148 ; 4-byte Folded Reload
@ -570,8 +571,8 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 4
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 5
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 3
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:12 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:16 ; 4-byte Folded Reload
@ -630,8 +631,8 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 8
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 9
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
; GFX9-O0-NEXT: s_or_b64 exec, exec, s[4:5]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:140 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:144 ; 4-byte Folded Reload
@ -661,8 +662,8 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 10
; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 11
; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 8
; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 9
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:228 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:232 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:236 ; 4-byte Folded Reload
@ -836,11 +837,11 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:176 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 6
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 7
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 4
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 5
; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 10
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 11
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
@ -964,8 +965,8 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v16, off, s[0:3], s32 offset:300 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:304 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 10
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 11
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
@ -1114,8 +1115,8 @@ define i128 @v_srem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[6:7], exec
; GFX9-O0-NEXT: s_and_b64 s[4:5], s[6:7], s[4:5]
; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 9
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 6
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 7
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
@ -1404,9 +1405,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_add_u32_e32 v9, 32, v9
; GFX9-NEXT: v_ffbh_u32_e32 v10, v5
; GFX9-NEXT: v_min_u32_e32 v9, v9, v10
; GFX9-NEXT: s_or_b64 s[4:5], vcc, s[4:5]
; GFX9-NEXT: s_or_b64 s[6:7], vcc, s[4:5]
; GFX9-NEXT: v_add_co_u32_e32 v9, vcc, 64, v9
; GFX9-NEXT: v_addc_co_u32_e64 v10, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v10, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[6:7]
; GFX9-NEXT: v_ffbh_u32_e32 v11, v3
; GFX9-NEXT: v_cndmask_b32_e32 v8, v9, v8, vcc
@ -1419,39 +1420,42 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_min_u32_e32 v11, v11, v12
; GFX9-NEXT: v_cndmask_b32_e64 v10, v10, 0, vcc
; GFX9-NEXT: v_add_co_u32_e32 v11, vcc, 64, v11
; GFX9-NEXT: v_addc_co_u32_e64 v12, s[6:7], 0, 0, vcc
; GFX9-NEXT: v_addc_co_u32_e64 v12, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[2:3]
; GFX9-NEXT: s_mov_b64 s[6:7], 0x7f
; GFX9-NEXT: s_subb_u32 s4, 0, 0
; GFX9-NEXT: v_cndmask_b32_e32 v9, v11, v9, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v12, v12, 0, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v8, vcc, v8, v9
; GFX9-NEXT: v_subb_co_u32_e32 v9, vcc, v10, v12, vcc
; GFX9-NEXT: v_mov_b32_e32 v11, 0
; GFX9-NEXT: v_subb_co_u32_e32 v10, vcc, 0, v11, vcc
; GFX9-NEXT: v_subb_co_u32_e32 v11, vcc, 0, v11, vcc
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[6:7], v[8:9]
; GFX9-NEXT: v_cndmask_b32_e64 v12, 0, 1, vcc
; GFX9-NEXT: s_subb_u32 s5, 0, 0
; GFX9-NEXT: s_mov_b64 s[8:9], 0x7f
; GFX9-NEXT: v_cmp_lt_u64_e32 vcc, s[8:9], v[8:9]
; GFX9-NEXT: s_cmp_lg_u64 s[4:5], 0
; GFX9-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-NEXT: s_cmp_eq_u64 s[4:5], 0
; GFX9-NEXT: v_cndmask_b32_e64 v10, 0, 1, vcc
; GFX9-NEXT: v_cndmask_b32_e64 v11, 0, 1, s[8:9]
; GFX9-NEXT: s_cselect_b64 vcc, -1, 0
; GFX9-NEXT: v_cndmask_b32_e32 v10, v11, v10, vcc
; GFX9-NEXT: v_and_b32_e32 v10, 1, v10
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v10
; GFX9-NEXT: v_xor_b32_e32 v10, 0x7f, v8
; GFX9-NEXT: v_or_b32_e32 v11, s5, v9
; GFX9-NEXT: v_or_b32_e32 v10, s4, v10
; GFX9-NEXT: s_or_b64 s[6:7], s[6:7], vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[10:11]
; GFX9-NEXT: v_cndmask_b32_e64 v13, 0, 1, vcc
; GFX9-NEXT: v_cmp_eq_u64_e32 vcc, 0, v[10:11]
; GFX9-NEXT: v_cndmask_b32_e32 v12, v13, v12, vcc
; GFX9-NEXT: v_and_b32_e32 v12, 1, v12
; GFX9-NEXT: v_cmp_eq_u32_e32 vcc, 1, v12
; GFX9-NEXT: v_xor_b32_e32 v12, 0x7f, v8
; GFX9-NEXT: v_or_b32_e32 v13, v9, v11
; GFX9-NEXT: v_or_b32_e32 v12, v12, v10
; GFX9-NEXT: s_or_b64 s[4:5], s[4:5], vcc
; GFX9-NEXT: v_cmp_ne_u64_e32 vcc, 0, v[12:13]
; GFX9-NEXT: s_xor_b64 s[6:7], s[4:5], -1
; GFX9-NEXT: v_cndmask_b32_e64 v15, v3, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v14, v2, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v13, v1, 0, s[4:5]
; GFX9-NEXT: v_cndmask_b32_e64 v12, v0, 0, s[4:5]
; GFX9-NEXT: s_and_b64 s[4:5], s[6:7], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[4:5]
; GFX9-NEXT: s_xor_b64 s[8:9], s[6:7], -1
; GFX9-NEXT: v_cndmask_b32_e64 v15, v3, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v14, v2, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v10, v1, 0, s[6:7]
; GFX9-NEXT: v_cndmask_b32_e64 v11, v0, 0, s[6:7]
; GFX9-NEXT: s_and_b64 s[6:7], s[8:9], vcc
; GFX9-NEXT: s_and_saveexec_b64 s[8:9], s[6:7]
; GFX9-NEXT: s_cbranch_execz .LBB1_6
; GFX9-NEXT: ; %bb.1: ; %udiv-bb1
; GFX9-NEXT: v_mov_b32_e32 v11, s5
; GFX9-NEXT: v_add_co_u32_e32 v22, vcc, 1, v8
; GFX9-NEXT: v_mov_b32_e32 v10, s4
; GFX9-NEXT: v_addc_co_u32_e32 v23, vcc, 0, v9, vcc
; GFX9-NEXT: v_addc_co_u32_e32 v24, vcc, 0, v10, vcc
; GFX9-NEXT: v_sub_u32_e32 v15, 0x7f, v8
@ -1556,27 +1560,27 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-NEXT: v_lshlrev_b64 v[8:9], 1, v[10:11]
; GFX9-NEXT: v_lshrrev_b32_e32 v16, 31, v11
; GFX9-NEXT: v_or_b32_e32 v14, v14, v16
; GFX9-NEXT: v_or_b32_e32 v13, v13, v9
; GFX9-NEXT: v_or_b32_e32 v12, v12, v8
; GFX9-NEXT: v_or_b32_e32 v10, v13, v9
; GFX9-NEXT: v_or_b32_e32 v11, v12, v8
; GFX9-NEXT: .LBB1_6: ; %Flow3
; GFX9-NEXT: s_or_b64 exec, exec, s[8:9]
; GFX9-NEXT: v_mul_lo_u32 v19, v12, v7
; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v4, v12, 0
; GFX9-NEXT: v_mul_lo_u32 v19, v11, v7
; GFX9-NEXT: v_mad_u64_u32 v[7:8], s[4:5], v4, v11, 0
; GFX9-NEXT: v_mov_b32_e32 v9, 0
; GFX9-NEXT: v_mul_lo_u32 v18, v13, v6
; GFX9-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v5, v12, v[8:9]
; GFX9-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v12, v6, 0
; GFX9-NEXT: v_mul_lo_u32 v18, v10, v6
; GFX9-NEXT: v_mad_u64_u32 v[16:17], s[4:5], v5, v11, v[8:9]
; GFX9-NEXT: v_mad_u64_u32 v[12:13], s[4:5], v11, v6, 0
; GFX9-NEXT: v_mov_b32_e32 v8, v16
; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v4, v13, v[8:9]
; GFX9-NEXT: v_add3_u32 v11, v11, v19, v18
; GFX9-NEXT: v_mad_u64_u32 v[10:11], s[4:5], v14, v4, v[10:11]
; GFX9-NEXT: v_mad_u64_u32 v[8:9], s[4:5], v4, v10, v[8:9]
; GFX9-NEXT: v_add3_u32 v13, v13, v19, v18
; GFX9-NEXT: v_mad_u64_u32 v[11:12], s[4:5], v14, v4, v[12:13]
; GFX9-NEXT: v_add_co_u32_e32 v13, vcc, v17, v9
; GFX9-NEXT: v_mul_lo_u32 v6, v14, v5
; GFX9-NEXT: v_add_co_u32_e32 v14, vcc, v17, v9
; GFX9-NEXT: v_mul_lo_u32 v12, v15, v4
; GFX9-NEXT: v_addc_co_u32_e64 v15, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v13, v[14:15]
; GFX9-NEXT: v_add3_u32 v6, v12, v11, v6
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v10
; GFX9-NEXT: v_addc_co_u32_e64 v14, s[4:5], 0, 0, vcc
; GFX9-NEXT: v_mul_lo_u32 v15, v15, v4
; GFX9-NEXT: v_mad_u64_u32 v[4:5], s[4:5], v5, v10, v[13:14]
; GFX9-NEXT: v_add3_u32 v6, v15, v12, v6
; GFX9-NEXT: v_add_co_u32_e32 v4, vcc, v4, v11
; GFX9-NEXT: v_addc_co_u32_e32 v5, vcc, v5, v6, vcc
; GFX9-NEXT: v_sub_co_u32_e32 v0, vcc, v0, v7
; GFX9-NEXT: v_subb_co_u32_e32 v1, vcc, v1, v8, vcc
@ -1739,48 +1743,48 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b32 s10, s6
; GFX9-O0-NEXT: s_mov_b32 s11, s7
; GFX9-O0-NEXT: v_sub_co_u32_e32 v4, vcc, v4, v7
; GFX9-O0-NEXT: v_subb_co_u32_e32 v8, vcc, v5, v6, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s10
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s10
; GFX9-O0-NEXT: v_subb_co_u32_e32 v7, vcc, v5, v6, vcc
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s11
; GFX9-O0-NEXT: v_mov_b32_e32 v5, s11
; GFX9-O0-NEXT: v_subb_co_u32_e32 v6, vcc, v5, v6, vcc
; GFX9-O0-NEXT: s_subb_u32 s12, s10, s10
; GFX9-O0-NEXT: s_subb_u32 s8, s11, s11
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 def $sgpr12_sgpr13
; GFX9-O0-NEXT: s_mov_b32 s13, s8
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v8
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: buffer_store_dword v4, off, s[0:3], s32 offset:28 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v5, off, s[0:3], s32 offset:32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: ; kill: def $vgpr7 killed $vgpr7 def $vgpr7_vgpr8 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v8, v6
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_mov_b32_e32 v6, s12
; GFX9-O0-NEXT: v_mov_b32_e32 v7, s13
; GFX9-O0-NEXT: buffer_store_dword v6, off, s[0:3], s32 offset:20 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v8, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_cmp_eq_u64_e64 s[8:9], v[7:8], s[6:7]
; GFX9-O0-NEXT: s_mov_b64 s[12:13], 0x7f
; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[14:15], v[4:5], s[12:13]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v9, 0, 1, s[14:15]
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[14:15], v[7:8], s[6:7]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[14:15]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v9, s[8:9]
; GFX9-O0-NEXT: buffer_store_dword v7, off, s[0:3], s32 offset:24 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 s[14:15], 0x7f
; GFX9-O0-NEXT: v_cmp_gt_u64_e64 s[8:9], v[4:5], s[14:15]
; GFX9-O0-NEXT: v_cndmask_b32_e64 v7, 0, 1, s[8:9]
; GFX9-O0-NEXT: s_cmp_lg_u64 s[12:13], s[6:7]
; GFX9-O0-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, 0, 1, s[8:9]
; GFX9-O0-NEXT: s_cmp_eq_u64 s[12:13], s[6:7]
; GFX9-O0-NEXT: s_cselect_b64 s[8:9], -1, 0
; GFX9-O0-NEXT: v_cndmask_b32_e64 v6, v6, v7, s[8:9]
; GFX9-O0-NEXT: v_and_b32_e64 v6, 1, v6
; GFX9-O0-NEXT: v_cmp_eq_u32_e64 s[8:9], v6, 1
; GFX9-O0-NEXT: s_or_b64 s[8:9], s[4:5], s[8:9]
; GFX9-O0-NEXT: s_mov_b64 s[4:5], -1
; GFX9-O0-NEXT: s_xor_b64 s[4:5], s[8:9], s[4:5]
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5
; GFX9-O0-NEXT: s_mov_b32 s14, s13
; GFX9-O0-NEXT: v_xor_b32_e64 v6, v6, s14
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, s12
; GFX9-O0-NEXT: s_mov_b32 s16, s15
; GFX9-O0-NEXT: v_xor_b32_e64 v6, v6, s16
; GFX9-O0-NEXT: ; kill: def $sgpr14 killed $sgpr14 killed $sgpr14_sgpr15
; GFX9-O0-NEXT: v_xor_b32_e64 v4, v4, s14
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_mov_b32_e32 v6, v5
; GFX9-O0-NEXT: v_mov_b32_e32 v9, v8
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, v9
; GFX9-O0-NEXT: s_mov_b32 s14, s13
; GFX9-O0-NEXT: v_or_b32_e64 v6, v6, s14
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 killed $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v7
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, v5
; GFX9-O0-NEXT: ; kill: def $sgpr12 killed $sgpr12 killed $sgpr12_sgpr13
; GFX9-O0-NEXT: v_or_b32_e64 v4, v4, s12
; GFX9-O0-NEXT: ; kill: def $vgpr4 killed $vgpr4 def $vgpr4_vgpr5 killed $exec
; GFX9-O0-NEXT: v_mov_b32_e32 v5, v6
; GFX9-O0-NEXT: v_cmp_ne_u64_e64 s[6:7], v[4:5], s[6:7]
@ -1806,17 +1810,17 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[4:5], exec
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 2
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 3
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_and_b64 s[4:5], s[4:5], s[6:7]
; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-O0-NEXT: s_cbranch_execz .LBB1_3
; GFX9-O0-NEXT: s_branch .LBB1_8
; GFX9-O0-NEXT: .LBB1_1: ; %Flow
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 4
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 5
@ -1848,9 +1852,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:108 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_5
; GFX9-O0-NEXT: .LBB1_3: ; %Flow2
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 2
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 3
@ -1908,9 +1912,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:8 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_3
; GFX9-O0-NEXT: .LBB1_5: ; %Flow1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s4, v30, 6
; GFX9-O0-NEXT: v_readlane_b32 s5, v30, 7
@ -1939,9 +1943,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_branch .LBB1_4
; GFX9-O0-NEXT: .LBB1_6: ; %udiv-do-while
; GFX9-O0-NEXT: ; =>This Inner Loop Header: Depth=1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_waitcnt vmcnt(0)
; GFX9-O0-NEXT: v_readlane_b32 s6, v30, 8
; GFX9-O0-NEXT: v_readlane_b32 s7, v30, 9
@ -2123,9 +2127,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_mov_b64 s[6:7], s[4:5]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
@ -2154,9 +2158,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_cbranch_execnz .LBB1_6
; GFX9-O0-NEXT: s_branch .LBB1_1
; GFX9-O0-NEXT: .LBB1_7: ; %udiv-preheader
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: buffer_load_dword v0, off, s[0:3], s32 offset:296 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v1, off, s[0:3], s32 offset:300 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v2, off, s[0:3], s32 offset:304 ; 4-byte Folded Reload
@ -2248,9 +2252,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v17, off, s[0:3], s32 offset:292 ; 4-byte Folded Spill
; GFX9-O0-NEXT: v_writelane_b32 v30, s4, 8
; GFX9-O0-NEXT: v_writelane_b32 v30, s5, 9
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: buffer_store_dword v14, off, s[0:3], s32 offset:272 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_nop 0
; GFX9-O0-NEXT: buffer_store_dword v15, off, s[0:3], s32 offset:276 ; 4-byte Folded Spill
@ -2277,9 +2281,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: buffer_store_dword v1, off, s[0:3], s32 offset:220 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_branch .LBB1_6
; GFX9-O0-NEXT: .LBB1_8: ; %udiv-bb1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_load_dword v30, off, s[0:3], s32 ; 4-byte Folded Reload
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: buffer_load_dword v6, off, s[0:3], s32 offset:36 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v7, off, s[0:3], s32 offset:40 ; 4-byte Folded Reload
; GFX9-O0-NEXT: buffer_load_dword v10, off, s[0:3], s32 offset:44 ; 4-byte Folded Reload
@ -2398,9 +2402,9 @@ define i128 @v_urem_i128_vv(i128 %lhs, i128 %rhs) {
; GFX9-O0-NEXT: s_xor_b64 s[6:7], s[4:5], s[6:7]
; GFX9-O0-NEXT: v_writelane_b32 v30, s6, 6
; GFX9-O0-NEXT: v_writelane_b32 v30, s7, 7
; GFX9-O0-NEXT: s_or_saveexec_b64 s[18:19], -1
; GFX9-O0-NEXT: s_or_saveexec_b64 s[20:21], -1
; GFX9-O0-NEXT: buffer_store_dword v30, off, s[0:3], s32 ; 4-byte Folded Spill
; GFX9-O0-NEXT: s_mov_b64 exec, s[18:19]
; GFX9-O0-NEXT: s_mov_b64 exec, s[20:21]
; GFX9-O0-NEXT: s_mov_b64 exec, s[4:5]
; GFX9-O0-NEXT: s_cbranch_execz .LBB1_5
; GFX9-O0-NEXT: s_branch .LBB1_7