From 7516bc1a57d4e07c25f62b3d35573617648f82ca Mon Sep 17 00:00:00 2001 From: Simon Pilgrim Date: Thu, 26 Feb 2026 14:36:47 +0000 Subject: [PATCH] [X86] Clearout unused FALLBACK check prefixes from vector extension/concatenation tests. NFC. (#183527) These were used before update_llc_test_checks reported when we had lost check coverage - we've cleaned up a lot of check prefixes since then --- .../any_extend_vector_inreg_of_broadcast.ll | 1434 ++++++----- .../X86/vector-shuffle-concatenation.ll | 552 ++--- .../CodeGen/X86/zero_extend_vector_inreg.ll | 2200 ++++++++--------- .../zero_extend_vector_inreg_of_broadcast.ll | 1764 +++++++------ 4 files changed, 2887 insertions(+), 3063 deletions(-) diff --git a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll index 44cf4e897d62..ac2f283166a9 100644 --- a/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll +++ b/llvm/test/CodeGen/X86/any_extend_vector_inreg_of_broadcast.ll @@ -1,18 +1,18 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,FALLBACK0 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42,FALLBACK1 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1-ONLY,FALLBACK2 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW,FALLBACK3 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE,FALLBACK4 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST,FALLBACK5 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW,FALLBACK6 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST,FALLBACK7 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW,FALLBACK8 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST,FALLBACK9 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW,FALLBACK10 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST,FALLBACK11 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW,FALLBACK12 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST,FALLBACK13 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST define void @vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind { ; SSE2-LABEL: vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2: @@ -38,14 +38,14 @@ define void @vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2(ptr %in.ve ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,5,0,7,u,u,u,u,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,5,0,7,u,u,u,u,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2: ; AVX2: # %bb.0: @@ -126,14 +126,14 @@ define void @vec64_i8_widen_to_i16_factor2_broadcast_to_v4i16_factor4(ptr %in.ve ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_i8_widen_to_i16_factor2_broadcast_to_v4i16_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,9,0,11,0,13,0,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_i8_widen_to_i16_factor2_broadcast_to_v4i16_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,9,0,11,0,13,0,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_i8_widen_to_i16_factor2_broadcast_to_v4i16_factor4: ; AVX2: # %bb.0: @@ -213,14 +213,14 @@ define void @vec64_i8_widen_to_i32_factor4_broadcast_to_v2i32_factor2(ptr %in.ve ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_i8_widen_to_i32_factor4_broadcast_to_v2i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,9,10,11,0,13,14,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_i8_widen_to_i32_factor4_broadcast_to_v2i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,9,10,11,0,13,14,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_i8_widen_to_i32_factor4_broadcast_to_v2i32_factor2: ; AVX2: # %bb.0: @@ -293,14 +293,14 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,14,15,10,11,12,13,14,15] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,14,15,10,11,12,13,14,15] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2: ; AVX2: # %bb.0: @@ -384,19 +384,19 @@ define void @vec128_i8_widen_to_i16_factor2_broadcast_to_v8i16_factor8(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i8_widen_to_i16_factor2_broadcast_to_v8i16_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i8_widen_to_i16_factor2_broadcast_to_v8i16_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i8_widen_to_i16_factor2_broadcast_to_v8i16_factor8: ; AVX2: # %bb.0: @@ -488,17 +488,17 @@ define void @vec128_i8_widen_to_i32_factor4_broadcast_to_v4i32_factor4(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i8_widen_to_i32_factor4_broadcast_to_v4i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i8_widen_to_i32_factor4_broadcast_to_v4i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i8_widen_to_i32_factor4_broadcast_to_v4i32_factor4: ; AVX2: # %bb.0: @@ -586,17 +586,17 @@ define void @vec128_i8_widen_to_i64_factor8_broadcast_to_v2i64_factor2(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i8_widen_to_i64_factor8_broadcast_to_v2i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i8_widen_to_i64_factor8_broadcast_to_v2i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i8_widen_to_i64_factor8_broadcast_to_v2i64_factor2: ; AVX2: # %bb.0: @@ -686,18 +686,18 @@ define void @vec128_i16_widen_to_i32_factor2_broadcast_to_v4i32_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i16_widen_to_i32_factor2_broadcast_to_v4i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i16_widen_to_i32_factor2_broadcast_to_v4i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i16_widen_to_i32_factor2_broadcast_to_v4i32_factor4: ; AVX2: # %bb.0: @@ -813,17 +813,17 @@ define void @vec128_i16_widen_to_i64_factor4_broadcast_to_v2i64_factor2(ptr %in. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i16_widen_to_i64_factor4_broadcast_to_v2i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i16_widen_to_i64_factor4_broadcast_to_v2i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i16_widen_to_i64_factor4_broadcast_to_v2i64_factor2: ; AVX2: # %bb.0: @@ -929,17 +929,17 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2: ; AVX2-SLOW: # %bb.0: @@ -1067,26 +1067,26 @@ define void @vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16(ptr %in ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0] -; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0] +; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16: ; AVX2: # %bb.0: @@ -1195,24 +1195,24 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] -; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm2[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] +; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm2[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8: ; AVX2: # %bb.0: @@ -1319,24 +1319,24 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] -; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm2[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] +; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm2[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4: ; AVX2: # %bb.0: @@ -1440,22 +1440,22 @@ define void @vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovsxwq {{.*#+}} xmm3 = [18446744073709551360,18446744073709551615] -; AVX-NEXT: vpblendvb %xmm3, %xmm1, %xmm0, %xmm1 -; AVX-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm3 = [18446744073709551360,18446744073709551615] +; AVX1-NEXT: vpblendvb %xmm3, %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -1573,25 +1573,25 @@ define void @vec256_i16_widen_to_i32_factor2_broadcast_to_v8i32_factor8(ptr %in. ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i16_widen_to_i32_factor2_broadcast_to_v8i32_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15] -; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i16_widen_to_i32_factor2_broadcast_to_v8i32_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15] +; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i16_widen_to_i32_factor2_broadcast_to_v8i32_factor8: ; AVX2: # %bb.0: @@ -1694,22 +1694,22 @@ define void @vec256_i16_widen_to_i64_factor4_broadcast_to_v4i64_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i16_widen_to_i64_factor4_broadcast_to_v4i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i16_widen_to_i64_factor4_broadcast_to_v4i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i16_widen_to_i64_factor4_broadcast_to_v4i64_factor4: ; AVX2: # %bb.0: @@ -1810,21 +1810,21 @@ define void @vec256_i16_widen_to_i128_factor8_broadcast_to_v2i128_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i16_widen_to_i128_factor8_broadcast_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i16_widen_to_i128_factor8_broadcast_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i16_widen_to_i128_factor8_broadcast_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -1926,25 +1926,25 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7] +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4: ; AVX2: # %bb.0: @@ -2079,24 +2079,24 @@ define void @vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -2231,24 +2231,24 @@ define void @vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -2394,25 +2394,25 @@ define void @vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24(ptr %in ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm2 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm2[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3],xmm2[4],xmm1[4],xmm2[5],xmm1[5],xmm2[6],xmm1[6],xmm2[7],xmm1[7] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24: ; AVX2: # %bb.0: @@ -2529,23 +2529,23 @@ define void @vec384_i8_widen_to_i24_factor3_broadcast_to_v16i24_factor16(ptr %in ; SSE42-NEXT: movdqa %xmm2, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i24_factor3_broadcast_to_v16i24_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,15,3,4,15,6,7,15,9,10,15,12,13,15] -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i24_factor3_broadcast_to_v16i24_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,15,3,4,15,6,7,15,9,10,15,12,13,15] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i24_factor3_broadcast_to_v16i24_factor16: ; AVX2: # %bb.0: @@ -2662,22 +2662,22 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12: ; AVX2: # %bb.0: @@ -2791,23 +2791,23 @@ define void @vec384_i8_widen_to_i48_factor6_broadcast_to_v8i48_factor8(ptr %in.v ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i48_factor6_broadcast_to_v8i48_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,2,3,4,15,6,7,8,9,10,15,12,13,14] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i48_factor6_broadcast_to_v8i48_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,2,3,4,15,6,7,8,9,10,15,12,13,14] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i48_factor6_broadcast_to_v8i48_factor8: ; AVX2: # %bb.0: @@ -2924,22 +2924,22 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6: ; AVX2: # %bb.0: @@ -3052,23 +3052,23 @@ define void @vec384_i8_widen_to_i96_factor12_broadcast_to_v4i96_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i96_factor12_broadcast_to_v4i96_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,2,3,4,5,6,7,8,9,10,15,12,13,14] -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i96_factor12_broadcast_to_v4i96_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[15,0,1,2,3,4,5,6,7,8,9,10,15,12,13,14] +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,1,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i96_factor12_broadcast_to_v4i96_factor4: ; AVX2: # %bb.0: @@ -3192,25 +3192,25 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i ; SSE42-NEXT: movdqa %xmm4, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovsxwq {{.*#+}} xmm3 = [18446744073709551360,18446744073709551615] -; AVX-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm3 = [18446744073709551360,18446744073709551615] +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3: ; AVX2: # %bb.0: @@ -3320,20 +3320,20 @@ define void @vec384_i8_widen_to_i192_factor24_broadcast_to_v2i192_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm3, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i192_factor24_broadcast_to_v2i192_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615] -; AVX-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm1 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i192_factor24_broadcast_to_v2i192_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615] +; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i192_factor24_broadcast_to_v2i192_factor2: ; AVX2: # %bb.0: @@ -3445,23 +3445,23 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i ; SSE42-NEXT: movdqa %xmm2, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm2, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm2, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12: ; AVX2: # %bb.0: @@ -3571,22 +3571,22 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in. ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8: ; AVX2: # %bb.0: @@ -3697,21 +3697,21 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in. ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6: ; AVX2: # %bb.0: @@ -3822,23 +3822,23 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm2, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1] -; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,0,0,0] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3,4,5],xmm3[6],xmm1[7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1] +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[0,0,0,0] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm3[0],xmm1[1,2,3,4,5],xmm3[6],xmm1[7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4: ; AVX2: # %bb.0: @@ -3947,24 +3947,24 @@ define void @vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3(ptr %i ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm2[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3: ; AVX2: # %bb.0: @@ -4068,19 +4068,19 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr % ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2: ; AVX2: # %bb.0: @@ -4184,25 +4184,25 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in. ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7] +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6: ; AVX2: # %bb.0: @@ -4311,23 +4311,23 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm2, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4: ; AVX2: # %bb.0: @@ -4445,24 +4445,24 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm2[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3: ; AVX2: # %bb.0: @@ -4577,19 +4577,19 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2: ; AVX2: # %bb.0: @@ -4699,24 +4699,24 @@ define void @vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3(ptr %i ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1,2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3: ; AVX2: # %bb.0: @@ -4831,21 +4831,21 @@ define void @vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm1 -; AVX-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm1 +; AVX1-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[2],ymm1[2] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2: ; AVX2: # %bb.0: @@ -4941,21 +4941,21 @@ define void @vec512_i8_widen_to_i16_factor2_broadcast_to_v32i16_factor32(ptr %in ; SSE-NEXT: movdqa %xmm1, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i16_factor2_broadcast_to_v32i16_factor32: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i16_factor2_broadcast_to_v32i16_factor32: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i16_factor2_broadcast_to_v32i16_factor32: ; AVX2: # %bb.0: @@ -5031,20 +5031,20 @@ define void @vec512_i8_widen_to_i32_factor4_broadcast_to_v16i32_factor16(ptr %in ; SSE-NEXT: movdqa %xmm1, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i32_factor4_broadcast_to_v16i32_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i32_factor4_broadcast_to_v16i32_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i32_factor4_broadcast_to_v16i32_factor16: ; AVX2: # %bb.0: @@ -5120,20 +5120,20 @@ define void @vec512_i8_widen_to_i64_factor8_broadcast_to_v8i64_factor8(ptr %in.v ; SSE-NEXT: movdqa %xmm1, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i64_factor8_broadcast_to_v8i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i64_factor8_broadcast_to_v8i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i64_factor8_broadcast_to_v8i64_factor8: ; AVX2: # %bb.0: @@ -5208,19 +5208,19 @@ define void @vec512_i8_widen_to_i128_factor16_broadcast_to_v4i128_factor4(ptr %i ; SSE-NEXT: movdqa %xmm1, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i128_factor16_broadcast_to_v4i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i128_factor16_broadcast_to_v4i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i128_factor16_broadcast_to_v4i128_factor4: ; AVX2: # %bb.0: @@ -5296,21 +5296,21 @@ define void @vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2(ptr %i ; SSE-NEXT: movdqa %xmm2, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2: ; AVX2: # %bb.0: @@ -5501,21 +5501,21 @@ define void @vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2(ptr % ; SSE-NEXT: movdqa %xmm2, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2: ; AVX2: # %bb.0: @@ -5570,19 +5570,3 @@ define void @vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2(ptr % store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64 ret void } -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; AVX1-ONLY: {{.*}} -; FALLBACK0: {{.*}} -; FALLBACK1: {{.*}} -; FALLBACK10: {{.*}} -; FALLBACK11: {{.*}} -; FALLBACK12: {{.*}} -; FALLBACK13: {{.*}} -; FALLBACK2: {{.*}} -; FALLBACK3: {{.*}} -; FALLBACK4: {{.*}} -; FALLBACK5: {{.*}} -; FALLBACK6: {{.*}} -; FALLBACK7: {{.*}} -; FALLBACK8: {{.*}} -; FALLBACK9: {{.*}} diff --git a/llvm/test/CodeGen/X86/vector-shuffle-concatenation.ll b/llvm/test/CodeGen/X86/vector-shuffle-concatenation.ll index 925f8d510451..4e76c03a723a 100644 --- a/llvm/test/CodeGen/X86/vector-shuffle-concatenation.ll +++ b/llvm/test/CodeGen/X86/vector-shuffle-concatenation.ll @@ -1,14 +1,14 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,FALLBACK0 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42,FALLBACK1 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1-ONLY,FALLBACK2 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW,FALLBACK3 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE,FALLBACK4 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST,FALLBACK5 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW,FALLBACK6 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST,FALLBACK7 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW,FALLBACK8 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST,FALLBACK9 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512 define void @concat_a_to_shuf_of_a(ptr %a.ptr, ptr %dst) { ; SSE-LABEL: concat_a_to_shuf_of_a: @@ -19,14 +19,14 @@ define void @concat_a_to_shuf_of_a(ptr %a.ptr, ptr %dst) { ; SSE-NEXT: movdqa %xmm1, (%rsi) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_a_to_shuf_of_a: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rdi), %xmm0 -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 -; AVX-NEXT: vmovaps %ymm0, (%rsi) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: concat_a_to_shuf_of_a: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rdi), %xmm0 +; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm0 +; AVX1-NEXT: vmovaps %ymm0, (%rsi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_a_to_shuf_of_a: ; AVX2: # %bb.0: @@ -36,21 +36,13 @@ define void @concat_a_to_shuf_of_a(ptr %a.ptr, ptr %dst) { ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_a_to_shuf_of_a: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rdi), %xmm0 -; AVX512F-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,1] -; AVX512F-NEXT: vmovaps %ymm0, (%rsi) -; AVX512F-NEXT: vzeroupper -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_a_to_shuf_of_a: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rdi), %xmm0 -; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,1] -; AVX512BW-NEXT: vmovaps %ymm0, (%rsi) -; AVX512BW-NEXT: vzeroupper -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_a_to_shuf_of_a: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rdi), %xmm0 +; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,0,1] +; AVX512-NEXT: vmovaps %ymm0, (%rsi) +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> %concat = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> @@ -66,14 +58,14 @@ define void @concat_shuf_of_a_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; SSE-NEXT: movdqa %xmm1, 16(%rdx) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_shuf_of_a_to_a: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rdi), %xmm0 -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 -; AVX-NEXT: vmovaps %ymm0, (%rdx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: concat_shuf_of_a_to_a: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rdi), %xmm0 +; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm0 +; AVX1-NEXT: vmovaps %ymm0, (%rdx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_shuf_of_a_to_a: ; AVX2: # %bb.0: @@ -83,21 +75,13 @@ define void @concat_shuf_of_a_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_shuf_of_a_to_a: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rdi), %xmm0 -; AVX512F-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,0] -; AVX512F-NEXT: vmovaps %ymm0, (%rdx) -; AVX512F-NEXT: vzeroupper -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_shuf_of_a_to_a: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rdi), %xmm0 -; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,0] -; AVX512BW-NEXT: vmovaps %ymm0, (%rdx) -; AVX512BW-NEXT: vzeroupper -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_shuf_of_a_to_a: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rdi), %xmm0 +; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[0,1,1,0] +; AVX512-NEXT: vmovaps %ymm0, (%rdx) +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %b = load <2 x i64>, ptr %b.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> @@ -116,14 +100,14 @@ define void @concat_a_to_shuf_of_a_extrause_of_shuf(ptr %a.ptr, ptr %dst, ptr %s ; SSE-NEXT: movdqa %xmm1, (%rsi) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rdi), %xmm0 -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX-NEXT: vmovaps %xmm1, (%rdx) -; AVX-NEXT: vmovaps %xmm0, 16(%rsi) -; AVX-NEXT: vmovaps %xmm1, (%rsi) -; AVX-NEXT: retq +; AVX1-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rdi), %xmm0 +; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vmovaps %xmm1, (%rdx) +; AVX1-NEXT: vmovaps %xmm0, 16(%rsi) +; AVX1-NEXT: vmovaps %xmm1, (%rsi) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf: ; AVX2: # %bb.0: @@ -134,23 +118,14 @@ define void @concat_a_to_shuf_of_a_extrause_of_shuf(ptr %a.ptr, ptr %dst, ptr %s ; AVX2-NEXT: vmovaps %xmm1, (%rsi) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rdi), %xmm0 -; AVX512F-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX512F-NEXT: vmovaps %xmm1, (%rdx) -; AVX512F-NEXT: vmovaps %xmm0, 16(%rsi) -; AVX512F-NEXT: vmovaps %xmm1, (%rsi) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rdi), %xmm0 -; AVX512BW-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX512BW-NEXT: vmovaps %xmm1, (%rdx) -; AVX512BW-NEXT: vmovaps %xmm0, 16(%rsi) -; AVX512BW-NEXT: vmovaps %xmm1, (%rsi) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_a_to_shuf_of_a_extrause_of_shuf: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rdi), %xmm0 +; AVX512-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vmovaps %xmm1, (%rdx) +; AVX512-NEXT: vmovaps %xmm0, 16(%rsi) +; AVX512-NEXT: vmovaps %xmm1, (%rsi) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> store <2 x i64> %shuffle, ptr %shuf.escape.ptr, align 64 @@ -178,13 +153,13 @@ define void @concat_a_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; SSE42-NEXT: movaps %xmm1, (%rdx) ; SSE42-NEXT: retq ; -; AVX-LABEL: concat_a_to_shuf_of_ab: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rdi), %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] -; AVX-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX-NEXT: vmovaps %xmm1, (%rdx) -; AVX-NEXT: retq +; AVX1-LABEL: concat_a_to_shuf_of_ab: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rdi), %xmm0 +; AVX1-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] +; AVX1-NEXT: vmovaps %xmm0, 16(%rdx) +; AVX1-NEXT: vmovaps %xmm1, (%rdx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_a_to_shuf_of_ab: ; AVX2: # %bb.0: @@ -194,21 +169,13 @@ define void @concat_a_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; AVX2-NEXT: vmovaps %xmm1, (%rdx) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_a_to_shuf_of_ab: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rdi), %xmm0 -; AVX512F-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] -; AVX512F-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX512F-NEXT: vmovaps %xmm1, (%rdx) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_a_to_shuf_of_ab: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rdi), %xmm0 -; AVX512BW-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] -; AVX512BW-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX512BW-NEXT: vmovaps %xmm1, (%rdx) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_a_to_shuf_of_ab: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rdi), %xmm0 +; AVX512-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] +; AVX512-NEXT: vmovaps %xmm0, 16(%rdx) +; AVX512-NEXT: vmovaps %xmm1, (%rdx) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %b = load <2 x i64>, ptr %b.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> @@ -235,13 +202,13 @@ define void @concat_b_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; SSE42-NEXT: movaps %xmm1, (%rdx) ; SSE42-NEXT: retq ; -; AVX-LABEL: concat_b_to_shuf_of_ab: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rsi), %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] -; AVX-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX-NEXT: vmovaps %xmm1, (%rdx) -; AVX-NEXT: retq +; AVX1-LABEL: concat_b_to_shuf_of_ab: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rsi), %xmm0 +; AVX1-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] +; AVX1-NEXT: vmovaps %xmm0, 16(%rdx) +; AVX1-NEXT: vmovaps %xmm1, (%rdx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_b_to_shuf_of_ab: ; AVX2: # %bb.0: @@ -251,21 +218,13 @@ define void @concat_b_to_shuf_of_ab(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; AVX2-NEXT: vmovaps %xmm1, (%rdx) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_b_to_shuf_of_ab: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rsi), %xmm0 -; AVX512F-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] -; AVX512F-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX512F-NEXT: vmovaps %xmm1, (%rdx) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_b_to_shuf_of_ab: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rsi), %xmm0 -; AVX512BW-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] -; AVX512BW-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX512BW-NEXT: vmovaps %xmm1, (%rdx) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_b_to_shuf_of_ab: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rsi), %xmm0 +; AVX512-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] +; AVX512-NEXT: vmovaps %xmm0, 16(%rdx) +; AVX512-NEXT: vmovaps %xmm1, (%rdx) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %b = load <2 x i64>, ptr %b.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> @@ -293,13 +252,13 @@ define void @concat_shuf_of_ab_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; SSE42-NEXT: movaps %xmm0, (%rdx) ; SSE42-NEXT: retq ; -; AVX-LABEL: concat_shuf_of_ab_to_a: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rdi), %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] -; AVX-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX-NEXT: vmovaps %xmm0, (%rdx) -; AVX-NEXT: retq +; AVX1-LABEL: concat_shuf_of_ab_to_a: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rdi), %xmm0 +; AVX1-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] +; AVX1-NEXT: vmovaps %xmm1, 16(%rdx) +; AVX1-NEXT: vmovaps %xmm0, (%rdx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_shuf_of_ab_to_a: ; AVX2: # %bb.0: @@ -309,21 +268,13 @@ define void @concat_shuf_of_ab_to_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; AVX2-NEXT: vmovaps %xmm0, (%rdx) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_shuf_of_ab_to_a: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rdi), %xmm0 -; AVX512F-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] -; AVX512F-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX512F-NEXT: vmovaps %xmm0, (%rdx) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_shuf_of_ab_to_a: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rdi), %xmm0 -; AVX512BW-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] -; AVX512BW-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX512BW-NEXT: vmovaps %xmm0, (%rdx) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_shuf_of_ab_to_a: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rdi), %xmm0 +; AVX512-NEXT: vblendps {{.*#+}} xmm1 = xmm0[0,1],mem[2,3] +; AVX512-NEXT: vmovaps %xmm1, 16(%rdx) +; AVX512-NEXT: vmovaps %xmm0, (%rdx) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %b = load <2 x i64>, ptr %b.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> @@ -350,13 +301,13 @@ define void @concat_shuf_of_ab_to_b(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; SSE42-NEXT: movaps %xmm0, (%rdx) ; SSE42-NEXT: retq ; -; AVX-LABEL: concat_shuf_of_ab_to_b: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rsi), %xmm0 -; AVX-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] -; AVX-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX-NEXT: vmovaps %xmm0, (%rdx) -; AVX-NEXT: retq +; AVX1-LABEL: concat_shuf_of_ab_to_b: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rsi), %xmm0 +; AVX1-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] +; AVX1-NEXT: vmovaps %xmm1, 16(%rdx) +; AVX1-NEXT: vmovaps %xmm0, (%rdx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_shuf_of_ab_to_b: ; AVX2: # %bb.0: @@ -366,21 +317,13 @@ define void @concat_shuf_of_ab_to_b(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; AVX2-NEXT: vmovaps %xmm0, (%rdx) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_shuf_of_ab_to_b: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rsi), %xmm0 -; AVX512F-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] -; AVX512F-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX512F-NEXT: vmovaps %xmm0, (%rdx) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_shuf_of_ab_to_b: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rsi), %xmm0 -; AVX512BW-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] -; AVX512BW-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX512BW-NEXT: vmovaps %xmm0, (%rdx) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_shuf_of_ab_to_b: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rsi), %xmm0 +; AVX512-NEXT: vblendps {{.*#+}} xmm1 = mem[0,1],xmm0[2,3] +; AVX512-NEXT: vmovaps %xmm1, 16(%rdx) +; AVX512-NEXT: vmovaps %xmm0, (%rdx) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %b = load <2 x i64>, ptr %b.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> %b, <2 x i32> @@ -398,13 +341,13 @@ define void @concat_b_to_shuf_of_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; SSE-NEXT: movdqa %xmm1, (%rdx) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_b_to_shuf_of_a: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rsi), %xmm0 -; AVX-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX-NEXT: vmovaps %xmm1, (%rdx) -; AVX-NEXT: retq +; AVX1-LABEL: concat_b_to_shuf_of_a: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rsi), %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX1-NEXT: vmovaps %xmm0, 16(%rdx) +; AVX1-NEXT: vmovaps %xmm1, (%rdx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_b_to_shuf_of_a: ; AVX2: # %bb.0: @@ -414,21 +357,13 @@ define void @concat_b_to_shuf_of_a(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; AVX2-NEXT: vmovaps %xmm1, (%rdx) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_b_to_shuf_of_a: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rsi), %xmm0 -; AVX512F-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX512F-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX512F-NEXT: vmovaps %xmm1, (%rdx) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_b_to_shuf_of_a: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rsi), %xmm0 -; AVX512BW-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX512BW-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX512BW-NEXT: vmovaps %xmm1, (%rdx) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_b_to_shuf_of_a: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rsi), %xmm0 +; AVX512-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512-NEXT: vmovaps %xmm0, 16(%rdx) +; AVX512-NEXT: vmovaps %xmm1, (%rdx) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %b = load <2 x i64>, ptr %b.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> @@ -445,13 +380,13 @@ define void @concat_shuf_of_a_to_b(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; SSE-NEXT: movaps %xmm0, (%rdx) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_shuf_of_a_to_b: -; AVX: # %bb.0: -; AVX-NEXT: vmovaps (%rsi), %xmm0 -; AVX-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX-NEXT: vmovaps %xmm0, (%rdx) -; AVX-NEXT: retq +; AVX1-LABEL: concat_shuf_of_a_to_b: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovaps (%rsi), %xmm0 +; AVX1-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX1-NEXT: vmovaps %xmm1, 16(%rdx) +; AVX1-NEXT: vmovaps %xmm0, (%rdx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_shuf_of_a_to_b: ; AVX2: # %bb.0: @@ -461,21 +396,13 @@ define void @concat_shuf_of_a_to_b(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; AVX2-NEXT: vmovaps %xmm0, (%rdx) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_shuf_of_a_to_b: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rsi), %xmm0 -; AVX512F-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX512F-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX512F-NEXT: vmovaps %xmm0, (%rdx) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_shuf_of_a_to_b: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rsi), %xmm0 -; AVX512BW-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] -; AVX512BW-NEXT: vmovaps %xmm1, 16(%rdx) -; AVX512BW-NEXT: vmovaps %xmm0, (%rdx) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_shuf_of_a_to_b: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rsi), %xmm0 +; AVX512-NEXT: vpermilps {{.*#+}} xmm1 = mem[2,3,0,1] +; AVX512-NEXT: vmovaps %xmm1, 16(%rdx) +; AVX512-NEXT: vmovaps %xmm0, (%rdx) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %b = load <2 x i64>, ptr %b.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> @@ -491,11 +418,11 @@ define void @concat_poison_to_shuf_of_a(ptr %a.ptr, ptr %dst) { ; SSE-NEXT: movdqa %xmm0, (%rsi) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_poison_to_shuf_of_a: -; AVX: # %bb.0: -; AVX-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] -; AVX-NEXT: vmovaps %xmm0, (%rsi) -; AVX-NEXT: retq +; AVX1-LABEL: concat_poison_to_shuf_of_a: +; AVX1: # %bb.0: +; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] +; AVX1-NEXT: vmovaps %xmm0, (%rsi) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_poison_to_shuf_of_a: ; AVX2: # %bb.0: @@ -503,17 +430,11 @@ define void @concat_poison_to_shuf_of_a(ptr %a.ptr, ptr %dst) { ; AVX2-NEXT: vmovaps %xmm0, (%rsi) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_poison_to_shuf_of_a: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] -; AVX512F-NEXT: vmovaps %xmm0, (%rsi) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_poison_to_shuf_of_a: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] -; AVX512BW-NEXT: vmovaps %xmm0, (%rsi) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_poison_to_shuf_of_a: +; AVX512: # %bb.0: +; AVX512-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] +; AVX512-NEXT: vmovaps %xmm0, (%rsi) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> %concat = shufflevector <2 x i64> %shuffle, <2 x i64> poison, <4 x i32> @@ -527,11 +448,11 @@ define void @concat_shuf_of_a_to_poison(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; SSE-NEXT: movdqa %xmm0, 16(%rdx) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_shuf_of_a_to_poison: -; AVX: # %bb.0: -; AVX-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] -; AVX-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX-NEXT: retq +; AVX1-LABEL: concat_shuf_of_a_to_poison: +; AVX1: # %bb.0: +; AVX1-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] +; AVX1-NEXT: vmovaps %xmm0, 16(%rdx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_shuf_of_a_to_poison: ; AVX2: # %bb.0: @@ -539,17 +460,11 @@ define void @concat_shuf_of_a_to_poison(ptr %a.ptr, ptr %b.ptr, ptr %dst) { ; AVX2-NEXT: vmovaps %xmm0, 16(%rdx) ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_shuf_of_a_to_poison: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] -; AVX512F-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_shuf_of_a_to_poison: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] -; AVX512BW-NEXT: vmovaps %xmm0, 16(%rdx) -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_shuf_of_a_to_poison: +; AVX512: # %bb.0: +; AVX512-NEXT: vpermilps {{.*#+}} xmm0 = mem[2,3,0,1] +; AVX512-NEXT: vmovaps %xmm0, 16(%rdx) +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %b = load <2 x i64>, ptr %b.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> @@ -566,13 +481,13 @@ define void @concat_shuf_of_a_to_itself(ptr %a.ptr, ptr %dst) { ; SSE-NEXT: movdqa %xmm0, (%rsi) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_shuf_of_a_to_itself: -; AVX: # %bb.0: -; AVX-NEXT: vpermilpd {{.*#+}} xmm0 = mem[1,0] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vmovaps %ymm0, (%rsi) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: concat_shuf_of_a_to_itself: +; AVX1: # %bb.0: +; AVX1-NEXT: vpermilpd {{.*#+}} xmm0 = mem[1,0] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vmovaps %ymm0, (%rsi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_shuf_of_a_to_itself: ; AVX2: # %bb.0: @@ -582,21 +497,13 @@ define void @concat_shuf_of_a_to_itself(ptr %a.ptr, ptr %dst) { ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_shuf_of_a_to_itself: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vmovaps (%rdi), %xmm0 -; AVX512F-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,1,0] -; AVX512F-NEXT: vmovaps %ymm0, (%rsi) -; AVX512F-NEXT: vzeroupper -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_shuf_of_a_to_itself: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vmovaps (%rdi), %xmm0 -; AVX512BW-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,1,0] -; AVX512BW-NEXT: vmovaps %ymm0, (%rsi) -; AVX512BW-NEXT: vzeroupper -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_shuf_of_a_to_itself: +; AVX512: # %bb.0: +; AVX512-NEXT: vmovaps (%rdi), %xmm0 +; AVX512-NEXT: vpermpd {{.*#+}} ymm0 = ymm0[1,0,1,0] +; AVX512-NEXT: vmovaps %ymm0, (%rsi) +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> %concat = shufflevector <2 x i64> %shuffle, <2 x i64> %shuffle, <4 x i32> @@ -615,15 +522,15 @@ define void @concat_aaa_to_shuf_of_a(ptr %a.ptr, ptr %dst) { ; SSE-NEXT: movdqa %xmm1, (%rsi) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_aaa_to_shuf_of_a: -; AVX: # %bb.0: -; AVX-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm1 -; AVX-NEXT: vmovaps %ymm0, 32(%rsi) -; AVX-NEXT: vmovaps %ymm1, (%rsi) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: concat_aaa_to_shuf_of_a: +; AVX1: # %bb.0: +; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] +; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm1, %ymm1 +; AVX1-NEXT: vmovaps %ymm0, 32(%rsi) +; AVX1-NEXT: vmovaps %ymm1, (%rsi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_aaa_to_shuf_of_a: ; AVX2: # %bb.0: @@ -634,25 +541,15 @@ define void @concat_aaa_to_shuf_of_a(ptr %a.ptr, ptr %dst) { ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_aaa_to_shuf_of_a: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1] -; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX512F-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm1 -; AVX512F-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; AVX512F-NEXT: vmovdqa64 %zmm0, (%rsi) -; AVX512F-NEXT: vzeroupper -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_aaa_to_shuf_of_a: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1] -; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX512BW-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm1 -; AVX512BW-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 -; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rsi) -; AVX512BW-NEXT: vzeroupper -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_aaa_to_shuf_of_a: +; AVX512: # %bb.0: +; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1] +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vinserti128 $1, %xmm0, %ymm1, %ymm1 +; AVX512-NEXT: vinserti64x4 $1, %ymm0, %zmm1, %zmm0 +; AVX512-NEXT: vmovdqa64 %zmm0, (%rsi) +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> %concat01 = shufflevector <2 x i64> %shuffle, <2 x i64> %a, <4 x i32> @@ -672,15 +569,15 @@ define void @concat_shuf_of_a_to_aaa(ptr %a.ptr, ptr %dst) { ; SSE-NEXT: movdqa %xmm1, 48(%rsi) ; SSE-NEXT: retq ; -; AVX-LABEL: concat_shuf_of_a_to_aaa: -; AVX: # %bb.0: -; AVX-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] -; AVX-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 -; AVX-NEXT: vmovaps %ymm0, (%rsi) -; AVX-NEXT: vmovaps %ymm1, 32(%rsi) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: concat_shuf_of_a_to_aaa: +; AVX1: # %bb.0: +; AVX1-NEXT: vbroadcastf128 {{.*#+}} ymm0 = mem[0,1,0,1] +; AVX1-NEXT: vshufps {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm0, %ymm1 +; AVX1-NEXT: vmovaps %ymm0, (%rsi) +; AVX1-NEXT: vmovaps %ymm1, 32(%rsi) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: concat_shuf_of_a_to_aaa: ; AVX2: # %bb.0: @@ -691,25 +588,15 @@ define void @concat_shuf_of_a_to_aaa(ptr %a.ptr, ptr %dst) { ; AVX2-NEXT: vzeroupper ; AVX2-NEXT: retq ; -; AVX512F-LABEL: concat_shuf_of_a_to_aaa: -; AVX512F: # %bb.0: -; AVX512F-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1] -; AVX512F-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX512F-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 -; AVX512F-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512F-NEXT: vmovdqa64 %zmm0, (%rsi) -; AVX512F-NEXT: vzeroupper -; AVX512F-NEXT: retq -; -; AVX512BW-LABEL: concat_shuf_of_a_to_aaa: -; AVX512BW: # %bb.0: -; AVX512BW-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1] -; AVX512BW-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] -; AVX512BW-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 -; AVX512BW-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 -; AVX512BW-NEXT: vmovdqa64 %zmm0, (%rsi) -; AVX512BW-NEXT: vzeroupper -; AVX512BW-NEXT: retq +; AVX512-LABEL: concat_shuf_of_a_to_aaa: +; AVX512: # %bb.0: +; AVX512-NEXT: vbroadcasti128 {{.*#+}} ymm0 = mem[0,1,0,1] +; AVX512-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[2,3,0,1] +; AVX512-NEXT: vinserti128 $1, %xmm1, %ymm0, %ymm1 +; AVX512-NEXT: vinserti64x4 $1, %ymm1, %zmm0, %zmm0 +; AVX512-NEXT: vmovdqa64 %zmm0, (%rsi) +; AVX512-NEXT: vzeroupper +; AVX512-NEXT: retq %a = load <2 x i64>, ptr %a.ptr, align 64 %shuffle = shufflevector <2 x i64> %a, <2 x i64> poison, <2 x i32> %concat01 = shufflevector <2 x i64> %a, <2 x i64> %a, <4 x i32> @@ -718,22 +605,3 @@ define void @concat_shuf_of_a_to_aaa(ptr %a.ptr, ptr %dst) { store <8 x i64> %concat, ptr %dst, align 64 ret void } -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; AVX1-ONLY: {{.*}} -; AVX2-FAST: {{.*}} -; AVX2-FAST-PERLANE: {{.*}} -; AVX2-SLOW: {{.*}} -; AVX512BW-FAST: {{.*}} -; AVX512BW-SLOW: {{.*}} -; AVX512F-FAST: {{.*}} -; AVX512F-SLOW: {{.*}} -; FALLBACK0: {{.*}} -; FALLBACK1: {{.*}} -; FALLBACK2: {{.*}} -; FALLBACK3: {{.*}} -; FALLBACK4: {{.*}} -; FALLBACK5: {{.*}} -; FALLBACK6: {{.*}} -; FALLBACK7: {{.*}} -; FALLBACK8: {{.*}} -; FALLBACK9: {{.*}} diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll index cacc43e96b6e..402a9387d81a 100644 --- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll +++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg.ll @@ -1,14 +1,14 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,FALLBACK0 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42,FALLBACK1 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1-ONLY,FALLBACK2 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW,FALLBACK3 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE,FALLBACK4 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST,FALLBACK5 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW,FALLBACK6 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST,FALLBACK7 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW,FALLBACK8 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST,FALLBACK9 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST define void @vec16_v2i8_to_v1i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind { ; SSE2-LABEL: vec16_v2i8_to_v1i16_factor2: @@ -32,14 +32,14 @@ define void @vec16_v2i8_to_v1i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec16_v2i8_to_v1i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec16_v2i8_to_v1i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec16_v2i8_to_v1i16_factor2: ; AVX2: # %bb.0: @@ -102,14 +102,14 @@ define void @vec32_v4i8_to_v2i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec32_v4i8_to_v2i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec32_v4i8_to_v2i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec32_v4i8_to_v2i16_factor2: ; AVX2: # %bb.0: @@ -174,14 +174,14 @@ define void @vec32_v4i8_to_v1i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec32_v4i8_to_v1i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec32_v4i8_to_v1i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec32_v4i8_to_v1i32_factor4: ; AVX2: # %bb.0: @@ -245,14 +245,14 @@ define void @vec32_v2i16_to_v1i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec32_v2i16_to_v1i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec32_v2i16_to_v1i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec32_v2i16_to_v1i32_factor2: ; AVX2: # %bb.0: @@ -317,14 +317,14 @@ define void @vec64_v8i8_to_v4i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_v8i8_to_v4i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_v8i8_to_v4i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_v8i8_to_v4i16_factor2: ; AVX2: # %bb.0: @@ -388,14 +388,14 @@ define void @vec64_v8i8_to_v2i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_v8i8_to_v2i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_v8i8_to_v2i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_v8i8_to_v2i32_factor4: ; AVX2: # %bb.0: @@ -460,14 +460,14 @@ define void @vec64_v8i8_to_v1i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bias. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_v8i8_to_v1i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_v8i8_to_v1i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_v8i8_to_v1i64_factor8: ; AVX2: # %bb.0: @@ -530,14 +530,14 @@ define void @vec64_v4i16_to_v2i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_v4i16_to_v2i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_v4i16_to_v2i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_v4i16_to_v2i32_factor2: ; AVX2: # %bb.0: @@ -603,14 +603,14 @@ define void @vec64_v4i16_to_v1i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bias ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_v4i16_to_v1i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_v4i16_to_v1i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_v4i16_to_v1i64_factor4: ; AVX2: # %bb.0: @@ -675,14 +675,14 @@ define void @vec64_v2i32_to_v1i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_v2i32_to_v1i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_v2i32_to_v1i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_v2i32_to_v1i64_factor2: ; AVX2: # %bb.0: @@ -747,14 +747,14 @@ define void @vec128_v16i8_to_v8i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_v16i8_to_v8i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v16i8_to_v8i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v16i8_to_v8i16_factor2: ; AVX2: # %bb.0: @@ -818,14 +818,14 @@ define void @vec128_v16i8_to_v4i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_v16i8_to_v4i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v16i8_to_v4i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v16i8_to_v4i32_factor4: ; AVX2: # %bb.0: @@ -890,14 +890,14 @@ define void @vec128_v16i8_to_v2i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_v16i8_to_v2i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v16i8_to_v2i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v16i8_to_v2i64_factor8: ; AVX2: # %bb.0: @@ -950,14 +950,14 @@ define void @vec128_v16i8_to_v1i128_factor16(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec128_v16i8_to_v1i128_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v16i8_to_v1i128_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v16i8_to_v1i128_factor16: ; AVX2: # %bb.0: @@ -1020,14 +1020,14 @@ define void @vec128_v8i16_to_v4i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_v8i16_to_v4i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v8i16_to_v4i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v8i16_to_v4i32_factor2: ; AVX2: # %bb.0: @@ -1093,14 +1093,14 @@ define void @vec128_v8i16_to_v2i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_v8i16_to_v2i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v8i16_to_v2i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v8i16_to_v2i64_factor4: ; AVX2: # %bb.0: @@ -1165,15 +1165,15 @@ define void @vec128_v8i16_to_v1i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_v8i16_to_v1i128_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v8i16_to_v1i128_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v8i16_to_v1i128_factor8: ; AVX2: # %bb.0: @@ -1241,14 +1241,14 @@ define void @vec128_v4i32_to_v2i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_v4i32_to_v2i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v4i32_to_v2i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v4i32_to_v2i64_factor2: ; AVX2: # %bb.0: @@ -1314,15 +1314,15 @@ define void @vec128_v4i32_to_v1i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_v4i32_to_v1i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v4i32_to_v1i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v4i32_to_v1i128_factor4: ; AVX2: # %bb.0: @@ -1380,14 +1380,14 @@ define void @vec128_v2i64_to_v1i128_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec128_v2i64_to_v1i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_v2i64_to_v1i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_v2i64_to_v1i128_factor2: ; AVX2: # %bb.0: @@ -1460,18 +1460,18 @@ define void @vec256_v32i8_to_v16i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v32i8_to_v16i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v32i8_to_v16i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm2[8],xmm0[9],xmm2[9],xmm0[10],xmm2[10],xmm0[11],xmm2[11],xmm0[12],xmm2[12],xmm0[13],xmm2[13],xmm0[14],xmm2[14],xmm0[15],xmm2[15] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v32i8_to_v16i16_factor2: ; AVX2: # %bb.0: @@ -1543,18 +1543,18 @@ define void @vec256_v32i8_to_v8i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v32i8_to_v8i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v32i8_to_v8i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v32i8_to_v8i32_factor4: ; AVX2: # %bb.0: @@ -1627,18 +1627,18 @@ define void @vec256_v32i8_to_v4i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v32i8_to_v4i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v32i8_to_v4i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v32i8_to_v4i64_factor8: ; AVX2: # %bb.0: @@ -1710,18 +1710,18 @@ define void @vec256_v32i8_to_v2i128_factor16(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v32i8_to_v2i128_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1] -; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v32i8_to_v2i128_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v32i8_to_v2i128_factor16: ; AVX2: # %bb.0: @@ -1782,16 +1782,16 @@ define void @vec256_v32i8_to_v1i256_factor32(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec256_v32i8_to_v1i256_factor32: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm1 -; AVX-NEXT: vmovaps %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v32i8_to_v1i256_factor32: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm1 +; AVX1-NEXT: vmovaps %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v32i8_to_v1i256_factor32: ; AVX2: # %bb.0: @@ -1865,18 +1865,18 @@ define void @vec256_v16i16_to_v8i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v16i16_to_v8i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v16i16_to_v8i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v16i16_to_v8i32_factor2: ; AVX2: # %bb.0: @@ -1950,18 +1950,18 @@ define void @vec256_v16i16_to_v4i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v16i16_to_v4i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v16i16_to_v4i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v16i16_to_v4i64_factor4: ; AVX2: # %bb.0: @@ -2035,19 +2035,19 @@ define void @vec256_v16i16_to_v2i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v16i16_to_v2i128_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] -; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v16i16_to_v2i128_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v16i16_to_v2i128_factor8: ; AVX2: # %bb.0: @@ -2124,17 +2124,17 @@ define void @vec256_v16i16_to_v1i256_factor16(ptr %in.vec.base.ptr, ptr %in.vec. ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v16i16_to_v1i256_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm1 -; AVX-NEXT: vmovaps %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v16i16_to_v1i256_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm1 +; AVX1-NEXT: vmovaps %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v16i16_to_v1i256_factor16: ; AVX2: # %bb.0: @@ -2210,18 +2210,18 @@ define void @vec256_v8i32_to_v4i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v8i32_to_v4i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v8i32_to_v4i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v8i32_to_v4i64_factor2: ; AVX2: # %bb.0: @@ -2297,18 +2297,18 @@ define void @vec256_v8i32_to_v2i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v8i32_to_v2i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[1],zero,zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v8i32_to_v2i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = xmm0[1],zero,zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec256_v8i32_to_v2i128_factor4: ; AVX2-SLOW: # %bb.0: @@ -2411,17 +2411,17 @@ define void @vec256_v8i32_to_v1i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_v8i32_to_v1i256_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm1 -; AVX-NEXT: vmovaps %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v8i32_to_v1i256_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm1 +; AVX1-NEXT: vmovaps %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v8i32_to_v1i256_factor8: ; AVX2: # %bb.0: @@ -2482,20 +2482,20 @@ define void @vec256_v4i64_to_v2i128_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE-NEXT: movdqa %xmm0, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec256_v4i64_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vxorpd %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[3] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v4i64_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vxorpd %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm1[1],ymm0[3],ymm1[3] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v4i64_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -2558,16 +2558,16 @@ define void @vec256_v4i64_to_v1i256_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec256_v4i64_to_v1i256_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm1 -; AVX-NEXT: vmovaps %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v4i64_to_v1i256_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm1 +; AVX1-NEXT: vmovaps %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v4i64_to_v1i256_factor4: ; AVX2: # %bb.0: @@ -2623,15 +2623,15 @@ define void @vec256_v2i128_to_v1i256_factor2(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec256_v2i128_to_v1i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm1 -; AVX-NEXT: vmovaps %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_v2i128_to_v1i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm1 +; AVX1-NEXT: vmovaps %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_v2i128_to_v1i256_factor2: ; AVX2: # %bb.0: @@ -2711,23 +2711,23 @@ define void @vec384_v48i8_to_v24i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v24i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15] -; AVX-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v24i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15] +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v48i8_to_v24i16_factor2: ; AVX2: # %bb.0: @@ -2819,20 +2819,20 @@ define void @vec384_v48i8_to_v16i24_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v16i24_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,xmm0[1],zero,zero,xmm0[2],zero,zero,xmm0[3],zero,zero,xmm0[4],zero,zero,xmm0[5] -; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[6],zero,zero,xmm0[7],zero,zero,xmm0[8],zero,zero,xmm0[9],zero,zero,xmm0[10],zero -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[11],zero,zero,xmm0[12],zero,zero,xmm0[13],zero,zero,xmm0[14],zero,zero,xmm0[15],zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v16i24_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,xmm0[1],zero,zero,xmm0[2],zero,zero,xmm0[3],zero,zero,xmm0[4],zero,zero,xmm0[5] +; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[6],zero,zero,xmm0[7],zero,zero,xmm0[8],zero,zero,xmm0[9],zero,zero,xmm0[10],zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[11],zero,zero,xmm0[12],zero,zero,xmm0[13],zero,zero,xmm0[14],zero,zero,xmm0[15],zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v48i8_to_v16i24_factor3: ; AVX2: # %bb.0: @@ -2925,22 +2925,22 @@ define void @vec384_v48i8_to_v12i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v12i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v12i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v48i8_to_v12i32_factor4: ; AVX2-SLOW: # %bb.0: @@ -3053,20 +3053,20 @@ define void @vec384_v48i8_to_v8i48_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v8i48_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero -; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[3],zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,xmm0[5],zero -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v8i48_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[3],zero,zero,zero,zero,zero,xmm0[4],zero,zero,zero,zero,zero,xmm0[5],zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[6],zero,zero,zero,zero,zero,xmm0[7],zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v48i8_to_v8i48_factor6: ; AVX2: # %bb.0: @@ -3164,22 +3164,22 @@ define void @vec384_v48i8_to_v6i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v6i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpsrld $16, %xmm0, %xmm2 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v6i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm2 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,1,1,1] +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v48i8_to_v6i64_factor8: ; AVX2-SLOW: # %bb.0: @@ -3296,20 +3296,20 @@ define void @vec384_v48i8_to_v4i96_factor12(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v4i96_factor12: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v4i96_factor12: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[2],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[3],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v48i8_to_v4i96_factor12: ; AVX2: # %bb.0: @@ -3405,22 +3405,22 @@ define void @vec384_v48i8_to_v3i128_factor16(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v3i128_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX-NEXT: vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1] -; AVX-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2] -; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v3i128_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v48i8_to_v3i128_factor16: ; AVX2-SLOW: # %bb.0: @@ -3541,20 +3541,20 @@ define void @vec384_v48i8_to_v2i192_factor24(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v2i192_factor24: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovaps %ymm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v2i192_factor24: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vmovaps 32(%rdx), %ymm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovaps %ymm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v48i8_to_v2i192_factor24: ; AVX2: # %bb.0: @@ -3621,19 +3621,19 @@ define void @vec384_v48i8_to_v1i384_factor48(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec384_v48i8_to_v1i384_factor48: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v48i8_to_v1i384_factor48: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v48i8_to_v1i384_factor48: ; AVX2: # %bb.0: @@ -3721,23 +3721,23 @@ define void @vec384_v24i16_to_v12i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v24i16_to_v12i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v24i16_to_v12i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v24i16_to_v12i32_factor2: ; AVX2: # %bb.0: @@ -3831,24 +3831,24 @@ define void @vec384_v24i16_to_v8i48_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v24i16_to_v8i48_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1] -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7] -; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2] -; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6],xmm3[7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7] -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm3, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v24i16_to_v8i48_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,1,1] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0],xmm2[1,2],xmm1[3],xmm2[4,5],xmm1[6],xmm2[7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,2,2] +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0],xmm3[1],xmm2[2,3],xmm3[4],xmm2[5,6],xmm3[7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7] +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm3, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v24i16_to_v8i48_factor3: ; AVX2-SLOW: # %bb.0: @@ -4010,22 +4010,22 @@ define void @vec384_v24i16_to_v6i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v24i16_to_v6i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v24i16_to_v6i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v24i16_to_v6i64_factor4: ; AVX2-SLOW: # %bb.0: @@ -4146,23 +4146,23 @@ define void @vec384_v24i16_to_v4i96_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm3, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v24i16_to_v4i96_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[2,3],zero,zero -; AVX-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4],xmm3[5,6,7] -; AVX-NEXT: vpsrld $16, %xmm0, %xmm0 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3,4,5,6,7] -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v24i16_to_v4i96_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm0[0,1],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[2,3],zero,zero +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4],xmm3[5,6,7] +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1],xmm0[2],xmm3[3,4,5,6,7] +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v24i16_to_v4i96_factor6: ; AVX2-SLOW: # %bb.0: @@ -4323,23 +4323,23 @@ define void @vec384_v24i16_to_v3i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v24i16_to_v3i128_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] -; AVX-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5] -; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v24i16_to_v3i128_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v24i16_to_v3i128_factor8: ; AVX2-SLOW: # %bb.0: @@ -4502,22 +4502,22 @@ define void @vec384_v24i16_to_v2i192_factor12(ptr %in.vec.base.ptr, ptr %in.vec. ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v24i16_to_v2i192_factor12: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6,7] -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v24i16_to_v2i192_factor12: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm1[0,1,2,3],xmm0[4],xmm1[5,6,7] +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v24i16_to_v2i192_factor12: ; AVX2: # %bb.0: @@ -4600,20 +4600,20 @@ define void @vec384_v24i16_to_v1i384_factor24(ptr %in.vec.base.ptr, ptr %in.vec. ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v24i16_to_v1i384_factor24: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v24i16_to_v1i384_factor24: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v24i16_to_v1i384_factor24: ; AVX2: # %bb.0: @@ -4702,23 +4702,23 @@ define void @vec384_v12i32_to_v6i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v12i32_to_v6i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] -; AVX-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v12i32_to_v6i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm1 = xmm1[0],zero,xmm1[1],zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v12i32_to_v6i64_factor2: ; AVX2: # %bb.0: @@ -4810,22 +4810,22 @@ define void @vec384_v12i32_to_v4i96_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v12i32_to_v4i96_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],zero,zero,xmm0[1] -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7] -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm3, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v12i32_to_v4i96_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertps {{.*#+}} xmm1 = xmm0[0],zero,zero,xmm0[1] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0,1,2,3],xmm0[4,5],xmm2[6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0,1],xmm0[2,3],xmm2[4,5,6,7] +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm3, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v12i32_to_v4i96_factor3: ; AVX2-SLOW: # %bb.0: @@ -4972,22 +4972,22 @@ define void @vec384_v12i32_to_v3i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v12i32_to_v3i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vinsertps {{.*#+}} xmm3 = xmm0[1],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm3, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v12i32_to_v3i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vinsertps {{.*#+}} xmm3 = xmm0[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[2,3,2,3] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm3, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v12i32_to_v3i128_factor4: ; AVX2-SLOW: # %bb.0: @@ -5132,21 +5132,21 @@ define void @vec384_v12i32_to_v2i192_factor6(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v12i32_to_v2i192_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vinsertps {{.*#+}} xmm0 = zero,zero,xmm0[1],zero -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovaps %ymm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v12i32_to_v2i192_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vinsertps {{.*#+}} xmm0 = zero,zero,xmm0[1],zero +; AVX1-NEXT: vmovaps 32(%rdx), %ymm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovaps %ymm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_v12i32_to_v2i192_factor6: ; AVX2-SLOW: # %bb.0: @@ -5262,20 +5262,20 @@ define void @vec384_v12i32_to_v1i384_factor12(ptr %in.vec.base.ptr, ptr %in.vec. ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v12i32_to_v1i384_factor12: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v12i32_to_v1i384_factor12: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v12i32_to_v1i384_factor12: ; AVX2: # %bb.0: @@ -5345,25 +5345,25 @@ define void @vec384_v6i64_to_v3i128_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE-NEXT: movdqa %xmm0, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec384_v6i64_to_v3i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vxorpd %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[3],ymm2[3] -; AVX-NEXT: vmovq {{.*#+}} xmm1 = xmm1[0],zero -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v6i64_to_v3i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vxorpd %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[3],ymm2[3] +; AVX1-NEXT: vmovq {{.*#+}} xmm1 = xmm1[0],zero +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v6i64_to_v3i128_factor2: ; AVX2: # %bb.0: @@ -5455,21 +5455,21 @@ define void @vec384_v6i64_to_v2i192_factor3(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_v6i64_to_v2i192_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1] -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v6i64_to_v2i192_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vperm2f128 {{.*#+}} ymm1 = zero,zero,ymm0[0,1] +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v6i64_to_v2i192_factor3: ; AVX2: # %bb.0: @@ -5538,19 +5538,19 @@ define void @vec384_v6i64_to_v1i384_factor6(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec384_v6i64_to_v1i384_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v6i64_to_v1i384_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v6i64_to_v1i384_factor6: ; AVX2: # %bb.0: @@ -5612,18 +5612,18 @@ define void @vec384_v3i128_to_v1i384_factor3(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec384_v3i128_to_v1i384_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_v3i128_to_v1i384_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_v3i128_to_v1i384_factor3: ; AVX2: # %bb.0: @@ -5714,26 +5714,26 @@ define void @vec512_v64i8_to_v32i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v64i8_to_v32i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15] -; AVX-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero -; AVX-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15] -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm4, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v64i8_to_v32i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero,xmm0[4],zero,xmm0[5],zero,xmm0[6],zero,xmm0[7],zero +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm0 = xmm0[8],xmm3[8],xmm0[9],xmm3[9],xmm0[10],xmm3[10],xmm0[11],xmm3[11],xmm0[12],xmm3[12],xmm0[13],xmm3[13],xmm0[14],xmm3[14],xmm0[15],xmm3[15] +; AVX1-NEXT: vpmovzxbw {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero,xmm1[4],zero,xmm1[5],zero,xmm1[6],zero,xmm1[7],zero +; AVX1-NEXT: vpunpckhbw {{.*#+}} xmm1 = xmm1[8],xmm3[8],xmm1[9],xmm3[9],xmm1[10],xmm3[10],xmm1[11],xmm3[11],xmm1[12],xmm3[12],xmm1[13],xmm3[13],xmm1[14],xmm3[14],xmm1[15],xmm3[15] +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm4, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v64i8_to_v32i16_factor2: ; AVX2: # %bb.0: @@ -5828,26 +5828,26 @@ define void @vec512_v64i8_to_v16i32_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v64i8_to_v16i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3] -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] -; AVX-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v64i8_to_v16i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero,xmm2[2],zero,zero,zero,xmm2[3],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero,xmm3[2],zero,zero,zero,xmm3[3],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX1-NEXT: vpmovzxbd {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero,xmm0[2],zero,zero,zero,xmm0[3],zero,zero,zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v64i8_to_v16i32_factor4: ; AVX2: # %bb.0: @@ -5943,26 +5943,26 @@ define void @vec512_v64i8_to_v8i64_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bia ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v64i8_to_v8i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpsrld $16, %xmm0, %xmm2 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1] -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpsrlq $48, %xmm0, %xmm0 -; AVX-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v64i8_to_v8i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrld $16, %xmm0, %xmm2 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,zero,zero,zero,zero,xmm2[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[1,1,1,1] +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,zero,zero,zero,zero,xmm3[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpsrlq $48, %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxbq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[1],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v64i8_to_v8i64_factor8: ; AVX2: # %bb.0: @@ -6057,26 +6057,26 @@ define void @vec512_v64i8_to_v4i128_factor16(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v64i8_to_v4i128_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 -; AVX-NEXT: vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1] -; AVX-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2] -; AVX-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] -; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v64i8_to_v4i128_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm1 +; AVX1-NEXT: vpslldq {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,0,0,0] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec512_v64i8_to_v4i128_factor16: ; AVX2-SLOW: # %bb.0: @@ -6254,22 +6254,22 @@ define void @vec512_v64i8_to_v2i256_factor32(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 32(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v64i8_to_v2i256_factor32: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1] -; AVX-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v64i8_to_v2i256_factor32: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpslldq {{.*#+}} xmm1 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v64i8_to_v2i256_factor32: ; AVX2: # %bb.0: @@ -6337,19 +6337,19 @@ define void @vec512_v64i8_to_v1i512_factor64(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_v64i8_to_v1i512_factor64: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v64i8_to_v1i512_factor64: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v64i8_to_v1i512_factor64: ; AVX2: # %bb.0: @@ -6442,26 +6442,26 @@ define void @vec512_v32i16_to_v16i32_factor2(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v32i16_to_v16i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero -; AVX-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm4, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v32i16_to_v16i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm0 = xmm0[4],xmm3[4],xmm0[5],xmm3[5],xmm0[6],xmm3[6],xmm0[7],xmm3[7] +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero,xmm1[2],zero,xmm1[3],zero +; AVX1-NEXT: vpunpckhwd {{.*#+}} xmm1 = xmm1[4],xmm3[4],xmm1[5],xmm3[5],xmm1[6],xmm3[6],xmm1[7],xmm3[7] +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm4, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v32i16_to_v16i32_factor2: ; AVX2: # %bb.0: @@ -6557,26 +6557,26 @@ define void @vec512_v32i16_to_v8i64_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v32i16_to_v8i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3] -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] -; AVX-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v32i16_to_v8i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm1 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[1,1,1,1] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm2 = xmm2[0],zero,zero,zero,xmm2[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm3 = xmm0[2,3,2,3] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm3 = xmm3[0],zero,zero,zero,xmm3[1],zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[3,3,3,3] +; AVX1-NEXT: vpmovzxwq {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[1],zero,zero,zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v32i16_to_v8i64_factor4: ; AVX2: # %bb.0: @@ -6671,27 +6671,27 @@ define void @vec512_v32i16_to_v4i128_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm3, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v32i16_to_v4i128_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] -; AVX-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5] -; AVX-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v32i16_to_v4i128_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm2 = xmm2[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpslldq {{.*#+}} xmm3 = zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm3 = xmm3[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec512_v32i16_to_v4i128_factor8: ; AVX2-SLOW: # %bb.0: @@ -6845,23 +6845,23 @@ define void @vec512_v32i16_to_v2i256_factor16(ptr %in.vec.base.ptr, ptr %in.vec. ; SSE42-NEXT: movdqa %xmm0, 32(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v32i16_to_v2i256_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0] -; AVX-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v32i16_to_v2i256_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[0,0,0,0] +; AVX1-NEXT: vpsrldq {{.*#+}} xmm1 = xmm1[14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v32i16_to_v2i256_factor16: ; AVX2: # %bb.0: @@ -6947,20 +6947,20 @@ define void @vec512_v32i16_to_v1i512_factor32(ptr %in.vec.base.ptr, ptr %in.vec. ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v32i16_to_v1i512_factor32: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v32i16_to_v1i512_factor32: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v32i16_to_v1i512_factor32: ; AVX2: # %bb.0: @@ -7054,26 +7054,26 @@ define void @vec512_v16i32_to_v8i64_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v16i32_to_v8i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] -; AVX-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero -; AVX-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3] -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm4, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v16i32_to_v8i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm2 = xmm0[0],zero,xmm0[1],zero +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm0 = xmm0[2],xmm3[2],xmm0[3],xmm3[3] +; AVX1-NEXT: vpmovzxdq {{.*#+}} xmm4 = xmm1[0],zero,xmm1[1],zero +; AVX1-NEXT: vpunpckhdq {{.*#+}} xmm1 = xmm1[2],xmm3[2],xmm1[3],xmm3[3] +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm4, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v16i32_to_v8i64_factor2: ; AVX2: # %bb.0: @@ -7170,24 +7170,24 @@ define void @vec512_v16i32_to_v4i128_factor4(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm3, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v16i32_to_v4i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vinsertps {{.*#+}} xmm2 = xmm0[1],zero,zero,zero -; AVX-NEXT: vinsertps {{.*#+}} xmm3 = xmm0[2],zero,zero,zero -; AVX-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v16i32_to_v4i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vinsertps {{.*#+}} xmm2 = xmm0[1],zero,zero,zero +; AVX1-NEXT: vinsertps {{.*#+}} xmm3 = xmm0[2],zero,zero,zero +; AVX1-NEXT: vpsrldq {{.*#+}} xmm0 = xmm0[12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec512_v16i32_to_v4i128_factor4: ; AVX2-SLOW: # %bb.0: @@ -7324,23 +7324,23 @@ define void @vec512_v16i32_to_v2i256_factor8(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE42-NEXT: movdqa %xmm0, 32(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v16i32_to_v2i256_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v16i32_to_v2i256_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm0[1,1,1,1] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm1[0,1],xmm2[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec512_v16i32_to_v2i256_factor8: ; AVX2-SLOW: # %bb.0: @@ -7471,20 +7471,20 @@ define void @vec512_v16i32_to_v1i512_factor16(ptr %in.vec.base.ptr, ptr %in.vec. ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_v16i32_to_v1i512_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v16i32_to_v1i512_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v16i32_to_v1i512_factor16: ; AVX2: # %bb.0: @@ -7555,29 +7555,29 @@ define void @vec512_v8i64_to_v4i128_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE-NEXT: movdqa %xmm0, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_v8i64_to_v4i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vxorpd %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[3],ymm2[3] -; AVX-NEXT: vinsertf128 $1, %xmm1, %ymm1, %ymm1 -; AVX-NEXT: vshufpd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[3],ymm2[3] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm2 -; AVX-NEXT: vpaddb 48(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 48(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v8i64_to_v4i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vxorpd %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vshufpd {{.*#+}} ymm0 = ymm0[0],ymm2[1],ymm0[3],ymm2[3] +; AVX1-NEXT: vinsertf128 $1, %xmm1, %ymm1, %ymm1 +; AVX1-NEXT: vshufpd {{.*#+}} ymm1 = ymm1[0],ymm2[1],ymm1[3],ymm2[3] +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm2 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 48(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v8i64_to_v4i128_factor2: ; AVX2: # %bb.0: @@ -7670,21 +7670,21 @@ define void @vec512_v8i64_to_v2i256_factor4(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE-NEXT: movdqa %xmm1, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_v8i64_to_v2i256_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpsrldq {{.*#+}} xmm1 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v8i64_to_v2i256_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpsrldq {{.*#+}} xmm1 = xmm0[8,9,10,11,12,13,14,15],zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v8i64_to_v2i256_factor4: ; AVX2: # %bb.0: @@ -7753,19 +7753,19 @@ define void @vec512_v8i64_to_v1i512_factor8(ptr %in.vec.base.ptr, ptr %in.vec.bi ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_v8i64_to_v1i512_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v8i64_to_v1i512_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v8i64_to_v1i512_factor8: ; AVX2: # %bb.0: @@ -7829,21 +7829,21 @@ define void @vec512_v4i128_to_v2i256_factor2(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_v4i128_to_v2i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v4i128_to_v2i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v4i128_to_v2i256_factor2: ; AVX2: # %bb.0: @@ -7911,18 +7911,18 @@ define void @vec512_v4i128_to_v1i512_factor4(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm0, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_v4i128_to_v1i512_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 32(%rdx), %ymm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %ymm1, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v4i128_to_v1i512_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %ymm1, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v4i128_to_v1i512_factor4: ; AVX2: # %bb.0: @@ -7983,20 +7983,20 @@ define void @vec512_v2i256_to_v1i512_factor2(ptr %in.vec.base.ptr, ptr %in.vec.b ; SSE-NEXT: movdqa %xmm1, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_v2i256_to_v1i512_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps %ymm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_v2i256_to_v1i512_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vmovaps 32(%rdx), %ymm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %ymm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_v2i256_to_v1i512_factor2: ; AVX2: # %bb.0: @@ -8039,15 +8039,3 @@ define void @vec512_v2i256_to_v1i512_factor2(ptr %in.vec.base.ptr, ptr %in.vec.b store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64 ret void } -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; AVX1-ONLY: {{.*}} -; FALLBACK0: {{.*}} -; FALLBACK1: {{.*}} -; FALLBACK2: {{.*}} -; FALLBACK3: {{.*}} -; FALLBACK4: {{.*}} -; FALLBACK5: {{.*}} -; FALLBACK6: {{.*}} -; FALLBACK7: {{.*}} -; FALLBACK8: {{.*}} -; FALLBACK9: {{.*}} diff --git a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll index 31ed74535b12..bf2eb2b0f356 100644 --- a/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll +++ b/llvm/test/CodeGen/X86/zero_extend_vector_inreg_of_broadcast.ll @@ -1,18 +1,18 @@ ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py -; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2,FALLBACK0 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42,FALLBACK1 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX,AVX1-ONLY,FALLBACK2 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW,FALLBACK3 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE,FALLBACK4 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST,FALLBACK5 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW,FALLBACK6 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST,FALLBACK7 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW,FALLBACK8 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST,FALLBACK9 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW,FALLBACK10 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST,FALLBACK11 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW,FALLBACK12 -; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST,FALLBACK13 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse2 | FileCheck %s --check-prefixes=SSE,SSE2 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+sse4.2 | FileCheck %s --check-prefixes=SSE,SSE42 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx | FileCheck %s --check-prefixes=AVX1 +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2 | FileCheck %s --check-prefixes=AVX2,AVX2-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST-PERLANE +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx2,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX2,AVX2-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl | FileCheck %s --check-prefixes=AVX512F,AVX512F-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512F,AVX512F-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512DQ,AVX512DQ-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-SLOW +; RUN: llc < %s -mtriple=x86_64-- -mattr=+avx512vl,+avx512dq,+avx512bw,+fast-variable-crosslane-shuffle,+fast-variable-perlane-shuffle | FileCheck %s --check-prefixes=AVX512BW,AVX512BW-FAST define void @vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2(ptr %in.vec.base.ptr, ptr %in.vec.bias.ptr, ptr %out.vec.bias.ptr, ptr %out.vec.ptr) nounwind { ; SSE2-LABEL: vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2: @@ -38,14 +38,14 @@ define void @vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2(ptr %in.ve ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,5,0,7,u,u,u,u,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,5,0,7,u,u,u,u,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec32_i8_widen_to_i16_factor2_broadcast_to_v2i16_factor2: ; AVX2: # %bb.0: @@ -126,14 +126,14 @@ define void @vec64_i8_widen_to_i16_factor2_broadcast_to_v4i16_factor4(ptr %in.ve ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_i8_widen_to_i16_factor2_broadcast_to_v4i16_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,9,0,11,0,13,0,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_i8_widen_to_i16_factor2_broadcast_to_v4i16_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,9,0,11,0,13,0,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_i8_widen_to_i16_factor2_broadcast_to_v4i16_factor4: ; AVX2: # %bb.0: @@ -213,14 +213,14 @@ define void @vec64_i8_widen_to_i32_factor4_broadcast_to_v2i32_factor2(ptr %in.ve ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_i8_widen_to_i32_factor4_broadcast_to_v2i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,9,10,11,0,13,14,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_i8_widen_to_i32_factor4_broadcast_to_v2i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,9,10,11,0,13,14,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_i8_widen_to_i32_factor4_broadcast_to_v2i32_factor2: ; AVX2: # %bb.0: @@ -293,14 +293,14 @@ define void @vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,14,15,10,11,12,13,14,15] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1,10,11,0,1,14,15,14,15,10,11,12,13,14,15] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec64_i16_widen_to_i32_factor2_broadcast_to_v2i32_factor2: ; AVX2: # %bb.0: @@ -384,19 +384,19 @@ define void @vec128_i8_widen_to_i16_factor2_broadcast_to_v8i16_factor8(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i8_widen_to_i16_factor2_broadcast_to_v8i16_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i8_widen_to_i16_factor2_broadcast_to_v8i16_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[1,3,5,7,9,11,13,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i8_widen_to_i16_factor2_broadcast_to_v8i16_factor8: ; AVX2: # %bb.0: @@ -488,17 +488,17 @@ define void @vec128_i8_widen_to_i32_factor4_broadcast_to_v4i32_factor4(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i8_widen_to_i32_factor4_broadcast_to_v4i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i8_widen_to_i32_factor4_broadcast_to_v4i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i8_widen_to_i32_factor4_broadcast_to_v4i32_factor4: ; AVX2: # %bb.0: @@ -586,17 +586,17 @@ define void @vec128_i8_widen_to_i64_factor8_broadcast_to_v2i64_factor2(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i8_widen_to_i64_factor8_broadcast_to_v2i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i8_widen_to_i64_factor8_broadcast_to_v2i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i8_widen_to_i64_factor8_broadcast_to_v2i64_factor2: ; AVX2: # %bb.0: @@ -686,18 +686,18 @@ define void @vec128_i16_widen_to_i32_factor2_broadcast_to_v4i32_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i16_widen_to_i32_factor2_broadcast_to_v4i32_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i16_widen_to_i32_factor2_broadcast_to_v4i32_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i16_widen_to_i32_factor2_broadcast_to_v4i32_factor4: ; AVX2: # %bb.0: @@ -813,17 +813,17 @@ define void @vec128_i16_widen_to_i64_factor4_broadcast_to_v2i64_factor2(ptr %in. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i16_widen_to_i64_factor4_broadcast_to_v2i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i16_widen_to_i64_factor4_broadcast_to_v2i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec128_i16_widen_to_i64_factor4_broadcast_to_v2i64_factor2: ; AVX2: # %bb.0: @@ -929,17 +929,17 @@ define void @vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2(ptr %in. ; SSE42-NEXT: movdqa %xmm0, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 16(%rdi), %xmm1 -; AVX-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 16(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 16(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3],xmm0[4,5],xmm1[6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec128_i32_widen_to_i64_factor2_broadcast_to_v2i64_factor2: ; AVX2-SLOW: # %bb.0: @@ -1067,26 +1067,26 @@ define void @vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16(ptr %in ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0] -; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] -; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vmovq {{.*#+}} xmm3 = [1,3,5,7,9,11,13,15,0,0,0,0,0,0,0,0] +; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0,0,1,1,2,2,3,3,4,4,5,5,6,6,7,7] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3],xmm0[4],xmm1[4],xmm0[5],xmm1[5],xmm0[6],xmm1[6],xmm0[7],xmm1[7] +; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpunpcklbw {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3],xmm0[4],xmm2[4],xmm0[5],xmm2[5],xmm0[6],xmm2[6],xmm0[7],xmm2[7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i8_widen_to_i16_factor2_broadcast_to_v16i16_factor16: ; AVX2: # %bb.0: @@ -1195,24 +1195,24 @@ define void @vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] -; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm2[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,0,1,2,15,4,5,6,15,8,9,10,15,12,13,14] +; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm2[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i8_widen_to_i32_factor4_broadcast_to_v8i32_factor8: ; AVX2: # %bb.0: @@ -1319,24 +1319,24 @@ define void @vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4(ptr %in.v ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] -; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX-NEXT: vpalignr {{.*#+}} xmm0 = xmm2[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] -; AVX-NEXT: vpshufb %xmm3, %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpalignr {{.*#+}} xmm1 = xmm1[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [15,0,1,2,3,4,5,6,15,8,9,10,11,12,13,14] +; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpalignr {{.*#+}} xmm0 = xmm2[1,2,3,4,5,6,7,8,9,10,11,12,13,14,15],xmm0[0] +; AVX1-NEXT: vpshufb %xmm3, %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i8_widen_to_i64_factor8_broadcast_to_v4i64_factor4: ; AVX2: # %bb.0: @@ -1440,22 +1440,22 @@ define void @vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovsxwq {{.*#+}} xmm3 = [18446744073709551360,18446744073709551615] -; AVX-NEXT: vpblendvb %xmm3, %xmm1, %xmm0, %xmm1 -; AVX-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm3 = [18446744073709551360,18446744073709551615] +; AVX1-NEXT: vpblendvb %xmm3, %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i8_widen_to_i128_factor16_broadcast_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -1573,25 +1573,25 @@ define void @vec256_i16_widen_to_i32_factor2_broadcast_to_v8i32_factor8(ptr %in. ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i16_widen_to_i32_factor2_broadcast_to_v8i32_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15] -; AVX-NEXT: vpshufb %xmm3, %xmm1, %xmm1 -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] -; AVX-NEXT: vpshufb %xmm3, %xmm2, %xmm2 -; AVX-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i16_widen_to_i32_factor2_broadcast_to_v8i32_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [2,3,6,7,10,11,14,15,14,15,10,11,12,13,14,15] +; AVX1-NEXT: vpshufb %xmm3, %xmm1, %xmm1 +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm0[0],xmm1[0],xmm0[1],xmm1[1],xmm0[2],xmm1[2],xmm0[3],xmm1[3] +; AVX1-NEXT: vpshufb %xmm3, %xmm2, %xmm2 +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm0 = xmm0[0],xmm2[0],xmm0[1],xmm2[1],xmm0[2],xmm2[2],xmm0[3],xmm2[3] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i16_widen_to_i32_factor2_broadcast_to_v8i32_factor8: ; AVX2: # %bb.0: @@ -1694,22 +1694,22 @@ define void @vec256_i16_widen_to_i64_factor4_broadcast_to_v4i64_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i16_widen_to_i64_factor4_broadcast_to_v4i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i16_widen_to_i64_factor4_broadcast_to_v4i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i16_widen_to_i64_factor4_broadcast_to_v4i64_factor4: ; AVX2: # %bb.0: @@ -1810,21 +1810,21 @@ define void @vec256_i16_widen_to_i128_factor8_broadcast_to_v2i128_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i16_widen_to_i128_factor8_broadcast_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i16_widen_to_i128_factor8_broadcast_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i16_widen_to_i128_factor8_broadcast_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -1926,25 +1926,25 @@ define void @vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7] +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i32_widen_to_i64_factor2_broadcast_to_v4i64_factor4: ; AVX2: # %bb.0: @@ -2079,24 +2079,24 @@ define void @vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i32_widen_to_i128_factor4_broadcast_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -2231,24 +2231,24 @@ define void @vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 32(%rdi), %xmm1 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm2 -; AVX-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 32(%rdi), %xmm1 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm2 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm2, %ymm1, %ymm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec256_i64_widen_to_i128_factor2_broadcast_to_v2i128_factor2: ; AVX2: # %bb.0: @@ -2396,24 +2396,24 @@ define void @vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24(ptr %in ; SSE42-NEXT: movdqa %xmm2, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1] -; AVX-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0] -; AVX-NEXT: vpblendvb %xmm3, %xmm2, %xmm1, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1] +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [255,0,255,0,255,0,255,0,255,0,255,0,255,0,255,0] +; AVX1-NEXT: vpblendvb %xmm3, %xmm2, %xmm1, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i16_factor2_broadcast_to_v24i16_factor24: ; AVX2: # %bb.0: @@ -2543,25 +2543,25 @@ define void @vec384_i8_widen_to_i24_factor3_broadcast_to_v16i24_factor16(ptr %in ; SSE42-NEXT: movdqa %xmm3, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i24_factor3_broadcast_to_v16i24_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpshufb %xmm2, %xmm0, %xmm2 -; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [0,255,255,0,255,255,0,255,255,0,255,255,0,255,255,0] -; AVX-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i24_factor3_broadcast_to_v16i24_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpshufb %xmm2, %xmm0, %xmm2 +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,255,255,0,255,255,0,255,255,0,255,255,0,255,255,0] +; AVX1-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero,xmm0[0],zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i24_factor3_broadcast_to_v16i24_factor16: ; AVX2: # %bb.0: @@ -2689,23 +2689,23 @@ define void @vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12(ptr %in ; SSE42-NEXT: movdqa %xmm3, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] -; AVX-NEXT: vbroadcastss {{.*#+}} xmm3 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255] -; AVX-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] +; AVX1-NEXT: vbroadcastss {{.*#+}} xmm3 = [0,255,255,255,0,255,255,255,0,255,255,255,0,255,255,255] +; AVX1-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i32_factor4_broadcast_to_v12i32_factor12: ; AVX2: # %bb.0: @@ -2835,25 +2835,25 @@ define void @vec384_i8_widen_to_i48_factor6_broadcast_to_v8i48_factor8(ptr %in.v ; SSE42-NEXT: movdqa %xmm3, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i48_factor6_broadcast_to_v8i48_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1] -; AVX-NEXT: vmovdqa {{.*#+}} xmm3 = [0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255] -; AVX-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[0],zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,xmm0[0],zero -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i48_factor6_broadcast_to_v8i48_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm2[0,1,0,1] +; AVX1-NEXT: vmovdqa {{.*#+}} xmm3 = [0,255,255,255,255,255,0,255,255,255,255,255,0,255,255,255] +; AVX1-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,xmm0[0],zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,xmm0[0],zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i48_factor6_broadcast_to_v8i48_factor8: ; AVX2: # %bb.0: @@ -2981,23 +2981,23 @@ define void @vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6(ptr %in.v ; SSE42-NEXT: movdqa %xmm3, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1] -; AVX-NEXT: vpmovsxwq {{.*#+}} xmm3 = [18446744073709551360,18446744073709551360] -; AVX-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1] +; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm3 = [18446744073709551360,18446744073709551360] +; AVX1-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i64_factor8_broadcast_to_v6i64_factor6: ; AVX2: # %bb.0: @@ -3127,24 +3127,24 @@ define void @vec384_i8_widen_to_i96_factor12_broadcast_to_v4i96_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm3, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i96_factor12_broadcast_to_v4i96_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] -; AVX-NEXT: vpmovsxwd {{.*#+}} xmm3 = [4294967040,4294967295,4294967295,4294967040] -; AVX-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i96_factor12_broadcast_to_v4i96_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] +; AVX1-NEXT: vpmovsxwd {{.*#+}} xmm3 = [4294967040,4294967295,4294967295,4294967040] +; AVX1-NEXT: vpblendvb %xmm3, %xmm1, %xmm2, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm2 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i96_factor12_broadcast_to_v4i96_factor4: ; AVX2: # %bb.0: @@ -3271,22 +3271,22 @@ define void @vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615] -; AVX-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm1 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615] +; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i128_factor16_broadcast_to_v3i128_factor3: ; AVX2: # %bb.0: @@ -3417,23 +3417,23 @@ define void @vec384_i8_widen_to_i192_factor24_broadcast_to_v2i192_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm3, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i8_widen_to_i192_factor24_broadcast_to_v2i192_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615] -; AVX-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps %ymm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i8_widen_to_i192_factor24_broadcast_to_v2i192_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpmovsxwq {{.*#+}} xmm2 = [18446744073709551360,18446744073709551615] +; AVX1-NEXT: vpblendvb %xmm2, %xmm1, %xmm0, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vmovaps 32(%rdx), %ymm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %ymm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i8_widen_to_i192_factor24_broadcast_to_v2i192_factor2: ; AVX2: # %bb.0: @@ -3558,24 +3558,24 @@ define void @vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12(ptr %i ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u] -; AVX-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm1 = xmm1[2,3,6,7,10,11,14,15,u,u,u,u,u,u,u,u] +; AVX1-NEXT: vpshuflw {{.*#+}} xmm2 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpunpcklwd {{.*#+}} xmm1 = xmm2[0],xmm1[0],xmm2[1],xmm1[1],xmm2[2],xmm1[2],xmm2[3],xmm1[3] +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm2 = xmm2[0],zero,xmm2[1],zero,xmm2[2],zero,xmm2[3],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero,xmm0[0,1],zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i32_factor2_broadcast_to_v12i32_factor12: ; AVX2: # %bb.0: @@ -3746,25 +3746,25 @@ define void @vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8(ptr %in. ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7] -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2],xmm0[3],xmm1[4,5],xmm0[6],xmm1[7] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm2[0,1],xmm0[2],xmm2[3,4],xmm0[5],xmm2[6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm2[0],xmm0[1],xmm2[2,3],xmm0[4],xmm2[5,6],xmm0[7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i48_factor3_broadcast_to_v8i48_factor8: ; AVX2: # %bb.0: @@ -3945,23 +3945,23 @@ define void @vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6(ptr %in. ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3],xmm0[4],xmm2[5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_i16_widen_to_i64_factor4_broadcast_to_v6i64_factor6: ; AVX2-SLOW: # %bb.0: @@ -4171,25 +4171,25 @@ define void @vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3,4,5],xmm2[6],xmm1[7] -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1] -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,0,0] +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm2[0],xmm1[1,2,3,4,5],xmm2[6],xmm1[7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,0,1,1] +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1],xmm2[2],xmm3[3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i96_factor6_broadcast_to_v4i96_factor4: ; AVX2: # %bb.0: @@ -4368,22 +4368,22 @@ define void @vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm2[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i128_factor8_broadcast_to_v3i128_factor3: ; AVX2: # %bb.0: @@ -4506,22 +4506,22 @@ define void @vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2(ptr % ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps %ymm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vmovaps 32(%rdx), %ymm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1],zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %ymm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i16_widen_to_i192_factor12_broadcast_to_v2i192_factor2: ; AVX2: # %bb.0: @@ -4643,28 +4643,28 @@ define void @vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6(ptr %in. ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 -; AVX-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,0],ymm1[1,3],ymm2[4,4],ymm1[5,7] -; AVX-NEXT: vshufps {{.*#+}} xmm2 = xmm1[0,2,1,3] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7] -; AVX-NEXT: vextractf128 $1, %ymm1, %xmm1 -; AVX-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm2, %xmm2 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm2, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm2 +; AVX1-NEXT: vshufps {{.*#+}} ymm1 = ymm2[0,0],ymm1[1,3],ymm2[4,4],ymm1[5,7] +; AVX1-NEXT: vshufps {{.*#+}} xmm2 = xmm1[0,2,1,3] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3],xmm0[4,5],xmm3[6,7] +; AVX1-NEXT: vextractf128 $1, %ymm1, %xmm1 +; AVX1-NEXT: vpunpckldq {{.*#+}} xmm1 = xmm1[0],xmm3[0],xmm1[1],xmm3[1] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_i32_widen_to_i64_factor2_broadcast_to_v6i64_factor6: ; AVX2-SLOW: # %bb.0: @@ -4842,26 +4842,26 @@ define void @vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4(ptr %in. ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] -; AVX-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1] -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,1,1] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5],xmm3[6,7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm2, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm0, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm1 = xmm1[0,1,2,0] +; AVX1-NEXT: vpshufd {{.*#+}} xmm2 = xmm0[0,1,0,1] +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm3[2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[1,0,1,1] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm2 = xmm3[0,1,2,3],xmm2[4,5],xmm3[6,7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm2, %xmm2 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-SLOW-LABEL: vec384_i32_widen_to_i96_factor3_broadcast_to_v4i96_factor4: ; AVX2-SLOW: # %bb.0: @@ -5039,23 +5039,23 @@ define void @vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpxor %xmm2, %xmm2, %xmm2 -; AVX-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2,3,4,5,6,7] -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpxor %xmm2, %xmm2, %xmm2 +; AVX1-NEXT: vpblendw {{.*#+}} xmm3 = xmm0[0,1],xmm2[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm2[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i32_widen_to_i128_factor4_broadcast_to_v3i128_factor3: ; AVX2: # %bb.0: @@ -5173,24 +5173,24 @@ define void @vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpxor %xmm3, %xmm3, %xmm3 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5],xmm3[6,7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps %ymm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vmovaps 32(%rdx), %ymm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpxor %xmm3, %xmm3, %xmm3 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm3[0,1,2,3],xmm0[4,5],xmm3[6,7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %ymm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i32_widen_to_i192_factor6_broadcast_to_v2i192_factor2: ; AVX2: # %bb.0: @@ -5304,21 +5304,21 @@ define void @vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3(ptr %i ; SSE42-NEXT: movdqa %xmm2, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7] -; AVX-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i64_widen_to_i128_factor2_broadcast_to_v3i128_factor3: ; AVX2: # %bb.0: @@ -5446,22 +5446,22 @@ define void @vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm1, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vmovdqa 48(%rdi), %xmm1 -; AVX-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7] -; AVX-NEXT: vmovaps 32(%rdx), %ymm2 -; AVX-NEXT: vpaddb (%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps %ymm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, (%rcx) -; AVX-NEXT: vmovdqa %xmm0, 16(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vmovdqa 48(%rdi), %xmm1 +; AVX1-NEXT: vpaddb 48(%rsi), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpblendw {{.*#+}} xmm1 = xmm0[0,1,2,3],xmm1[4,5,6,7] +; AVX1-NEXT: vmovaps 32(%rdx), %ymm2 +; AVX1-NEXT: vpaddb (%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpslldq {{.*#+}} xmm0 = zero,zero,zero,zero,zero,zero,zero,zero,xmm0[0,1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps %ymm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, (%rcx) +; AVX1-NEXT: vmovdqa %xmm0, 16(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec384_i64_widen_to_i192_factor3_broadcast_to_v2i192_factor2: ; AVX2: # %bb.0: @@ -5592,20 +5592,20 @@ define void @vec512_i8_widen_to_i16_factor2_broadcast_to_v32i16_factor32(ptr %in ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i16_factor2_broadcast_to_v32i16_factor32: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i16_factor2_broadcast_to_v32i16_factor32: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero,xmm0[0],zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i16_factor2_broadcast_to_v32i16_factor32: ; AVX2: # %bb.0: @@ -5704,20 +5704,20 @@ define void @vec512_i8_widen_to_i32_factor4_broadcast_to_v16i32_factor16(ptr %in ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i32_factor4_broadcast_to_v16i32_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i32_factor4_broadcast_to_v16i32_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero,xmm0[0],zero,zero,zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i32_factor4_broadcast_to_v16i32_factor16: ; AVX2: # %bb.0: @@ -5816,20 +5816,20 @@ define void @vec512_i8_widen_to_i64_factor8_broadcast_to_v8i64_factor8(ptr %in.v ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i64_factor8_broadcast_to_v8i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i64_factor8_broadcast_to_v8i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufb {{.*#+}} xmm0 = xmm0[0],zero,zero,zero,zero,zero,zero,zero,xmm0[0],zero,zero,zero,zero,zero,zero,zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i64_factor8_broadcast_to_v8i64_factor8: ; AVX2: # %bb.0: @@ -5909,20 +5909,20 @@ define void @vec512_i8_widen_to_i128_factor16_broadcast_to_v4i128_factor4(ptr %i ; SSE-NEXT: movdqa %xmm1, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i128_factor16_broadcast_to_v4i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i128_factor16_broadcast_to_v4i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i128_factor16_broadcast_to_v4i128_factor4: ; AVX2: # %bb.0: @@ -6000,20 +6000,20 @@ define void @vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2(ptr %i ; SSE-NEXT: movdqa %xmm3, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpand {{\.?LCPI[0-9]+_[0-9]+}}(%rip), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i8_widen_to_i256_factor32_broadcast_to_v2i256_factor2: ; AVX2: # %bb.0: @@ -6114,21 +6114,21 @@ define void @vec512_i16_widen_to_i32_factor2_broadcast_to_v16i32_factor16(ptr %i ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i16_widen_to_i32_factor2_broadcast_to_v16i32_factor16: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] -; AVX-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i16_widen_to_i32_factor2_broadcast_to_v16i32_factor16: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshuflw {{.*#+}} xmm0 = xmm0[0,0,0,0,4,5,6,7] +; AVX1-NEXT: vpmovzxwd {{.*#+}} xmm0 = xmm0[0],zero,xmm0[1],zero,xmm0[2],zero,xmm0[3],zero +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i16_widen_to_i32_factor2_broadcast_to_v16i32_factor16: ; AVX2: # %bb.0: @@ -6231,22 +6231,22 @@ define void @vec512_i16_widen_to_i64_factor4_broadcast_to_v8i64_factor8(ptr %in. ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i16_widen_to_i64_factor4_broadcast_to_v8i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i16_widen_to_i64_factor4_broadcast_to_v8i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpshufd {{.*#+}} xmm0 = xmm0[0,1,0,1] +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3],xmm0[4],xmm1[5,6,7] +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i16_widen_to_i64_factor4_broadcast_to_v8i64_factor8: ; AVX2: # %bb.0: @@ -6347,21 +6347,21 @@ define void @vec512_i16_widen_to_i128_factor8_broadcast_to_v4i128_factor4(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i16_widen_to_i128_factor8_broadcast_to_v4i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 -; AVX-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm3, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm1, 48(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i16_widen_to_i128_factor8_broadcast_to_v4i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 48(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm2 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 48(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i16_widen_to_i128_factor8_broadcast_to_v4i128_factor4: ; AVX2: # %bb.0: @@ -6461,21 +6461,21 @@ define void @vec512_i16_widen_to_i256_factor16_broadcast_to_v2i256_factor2(ptr % ; SSE42-NEXT: movdqa %xmm3, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i16_widen_to_i256_factor16_broadcast_to_v2i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i16_widen_to_i256_factor16_broadcast_to_v2i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0],xmm1[1,2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i16_widen_to_i256_factor16_broadcast_to_v2i256_factor2: ; AVX2: # %bb.0: @@ -6579,25 +6579,25 @@ define void @vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8(ptr %in. ; SSE42-NEXT: movdqa %xmm1, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7] -; AVX-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 48(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,0],ymm1[1,3],ymm0[4,4],ymm1[5,7] +; AVX1-NEXT: vshufps {{.*#+}} ymm0 = ymm0[0,2,1,3,4,6,5,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 48(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i32_widen_to_i64_factor2_broadcast_to_v8i64_factor8: ; AVX2: # %bb.0: @@ -6705,24 +6705,24 @@ define void @vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4(ptr %i ; SSE42-NEXT: movdqa %xmm0, 16(%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 48(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0],ymm1[1,2,3],ymm0[4],ymm1[5,6,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 48(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i32_widen_to_i128_factor4_broadcast_to_v4i128_factor4: ; AVX2: # %bb.0: @@ -6826,21 +6826,21 @@ define void @vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2(ptr %i ; SSE42-NEXT: movdqa %xmm3, (%rcx) ; SSE42-NEXT: retq ; -; AVX-LABEL: vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpxor %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7] -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpxor %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vpblendw {{.*#+}} xmm0 = xmm0[0,1],xmm1[2,3,4,5,6,7] +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i32_widen_to_i256_factor8_broadcast_to_v2i256_factor2: ; AVX2: # %bb.0: @@ -6927,24 +6927,24 @@ define void @vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4(ptr %i ; SSE-NEXT: movdqa %xmm1, 16(%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 -; AVX-NEXT: vxorps %xmm1, %xmm1, %xmm1 -; AVX-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] -; AVX-NEXT: vextractf128 $1, %ymm0, %xmm1 -; AVX-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 -; AVX-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 16(%rcx) -; AVX-NEXT: vmovdqa %xmm3, 32(%rcx) -; AVX-NEXT: vmovdqa %xmm2, 48(%rcx) -; AVX-NEXT: vzeroupper -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vinsertf128 $1, %xmm0, %ymm0, %ymm0 +; AVX1-NEXT: vxorps %xmm1, %xmm1, %xmm1 +; AVX1-NEXT: vblendps {{.*#+}} ymm0 = ymm0[0,1],ymm1[2,3],ymm0[4,5],ymm1[6,7] +; AVX1-NEXT: vextractf128 $1, %ymm0, %xmm1 +; AVX1-NEXT: vpaddb 48(%rdx), %xmm1, %xmm2 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm3 +; AVX1-NEXT: vpaddb 16(%rdx), %xmm1, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 16(%rcx) +; AVX1-NEXT: vmovdqa %xmm3, 32(%rcx) +; AVX1-NEXT: vmovdqa %xmm2, 48(%rcx) +; AVX1-NEXT: vzeroupper +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i64_widen_to_i128_factor2_broadcast_to_v4i128_factor4: ; AVX2: # %bb.0: @@ -7030,20 +7030,20 @@ define void @vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2(ptr %i ; SSE-NEXT: movdqa %xmm3, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vmovq {{.*#+}} xmm0 = xmm0[0],zero +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i64_widen_to_i256_factor4_broadcast_to_v2i256_factor2: ; AVX2: # %bb.0: @@ -7138,19 +7138,19 @@ define void @vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2(ptr % ; SSE-NEXT: movdqa %xmm3, (%rcx) ; SSE-NEXT: retq ; -; AVX-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2: -; AVX: # %bb.0: -; AVX-NEXT: vmovdqa (%rdi), %xmm0 -; AVX-NEXT: vpaddb (%rsi), %xmm0, %xmm0 -; AVX-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 -; AVX-NEXT: vpaddb (%rdx), %xmm0, %xmm0 -; AVX-NEXT: vmovaps 16(%rdx), %xmm2 -; AVX-NEXT: vmovaps 48(%rdx), %xmm3 -; AVX-NEXT: vmovaps %xmm2, 16(%rcx) -; AVX-NEXT: vmovaps %xmm3, 48(%rcx) -; AVX-NEXT: vmovdqa %xmm0, (%rcx) -; AVX-NEXT: vmovdqa %xmm1, 32(%rcx) -; AVX-NEXT: retq +; AVX1-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2: +; AVX1: # %bb.0: +; AVX1-NEXT: vmovdqa (%rdi), %xmm0 +; AVX1-NEXT: vpaddb (%rsi), %xmm0, %xmm0 +; AVX1-NEXT: vpaddb 32(%rdx), %xmm0, %xmm1 +; AVX1-NEXT: vpaddb (%rdx), %xmm0, %xmm0 +; AVX1-NEXT: vmovaps 16(%rdx), %xmm2 +; AVX1-NEXT: vmovaps 48(%rdx), %xmm3 +; AVX1-NEXT: vmovaps %xmm2, 16(%rcx) +; AVX1-NEXT: vmovaps %xmm3, 48(%rcx) +; AVX1-NEXT: vmovdqa %xmm0, (%rcx) +; AVX1-NEXT: vmovdqa %xmm1, 32(%rcx) +; AVX1-NEXT: retq ; ; AVX2-LABEL: vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2: ; AVX2: # %bb.0: @@ -7213,19 +7213,3 @@ define void @vec512_i128_widen_to_i256_factor2_broadcast_to_v2i256_factor2(ptr % store <64 x i8> %out.vec, ptr %out.vec.ptr, align 64 ret void } -;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line: -; AVX1-ONLY: {{.*}} -; FALLBACK0: {{.*}} -; FALLBACK1: {{.*}} -; FALLBACK10: {{.*}} -; FALLBACK11: {{.*}} -; FALLBACK12: {{.*}} -; FALLBACK13: {{.*}} -; FALLBACK2: {{.*}} -; FALLBACK3: {{.*}} -; FALLBACK4: {{.*}} -; FALLBACK5: {{.*}} -; FALLBACK6: {{.*}} -; FALLBACK7: {{.*}} -; FALLBACK8: {{.*}} -; FALLBACK9: {{.*}}