From 8bf04e9f2a7dae36c2cdae8a955cc77ec9e253cb Mon Sep 17 00:00:00 2001
From: Simon Pilgrim <llvm-dev@redking.me.uk>
Date: Tue, 13 Sep 2022 11:22:54 +0100
Subject: [PATCH] [X86] Add GFNI test coverage for bitreverse codegen

We should be able to efficiently use the vector version for scalar bitreverse, like we do for XOP.
---
 llvm/test/CodeGen/X86/bitreverse.ll | 432 +++++++++++++++++++++++-----
 1 file changed, 365 insertions(+), 67 deletions(-)

diff --git a/llvm/test/CodeGen/X86/bitreverse.ll b/llvm/test/CodeGen/X86/bitreverse.ll
index c8fb2335874f..3c0f20ab7aa0 100644
--- a/llvm/test/CodeGen/X86/bitreverse.ll
+++ b/llvm/test/CodeGen/X86/bitreverse.ll
@@ -1,7 +1,8 @@
 ; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefix=X86
-; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefix=X64
-; RUN: llc < %s -mtriple=i686-unknown -mattr=+xop | FileCheck %s --check-prefixes=X86XOP
+; RUN: llc < %s -mtriple=i686-unknown | FileCheck %s --check-prefixes=CHECK,X86
+; RUN: llc < %s -mtriple=x86_64-unknown | FileCheck %s --check-prefixes=CHECK,X64
+; RUN: llc < %s -mtriple=i686-unknown -mattr=+xop | FileCheck %s --check-prefixes=CHECK,X86XOP
+; RUN: llc < %s -mtriple=x86_64-unknown -mattr=+avx512bw,+avx512vl,+gfni | FileCheck %s --check-prefixes=CHECK,GFNI
 
 ; These tests just check that the plumbing is in place for @llvm.bitreverse. The
 ; actual output is massive at the moment as llvm.bitreverse is not yet legal.
@@ -84,6 +85,12 @@ define <2 x i16> @test_bitreverse_v2i16(<2 x i16> %a) nounwind {
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    vpperm {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: test_bitreverse_v2i16:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    vpshufb {{.*#+}} xmm0 = xmm0[1,0,3,2,5,4,7,6,9,8,11,10,13,12,15,14]
+; GFNI-NEXT:    vgf2p8affineqb $0, {{\.?LCPI[0-9]+_[0-9]+}}(%rip){1to2}, %xmm0, %xmm0
+; GFNI-NEXT:    retq
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a)
   ret <2 x i16> %b
 }
@@ -162,6 +169,30 @@ define i64 @test_bitreverse_i64(i64 %a) nounwind {
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    vpextrd $1, %xmm0, %edx
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: test_bitreverse_i64:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    bswapq %rdi
+; GFNI-NEXT:    movq %rdi, %rax
+; GFNI-NEXT:    shrq $4, %rax
+; GFNI-NEXT:    movabsq $1085102592571150095, %rcx # imm = 0xF0F0F0F0F0F0F0F
+; GFNI-NEXT:    andq %rcx, %rax
+; GFNI-NEXT:    andq %rcx, %rdi
+; GFNI-NEXT:    shlq $4, %rdi
+; GFNI-NEXT:    orq %rax, %rdi
+; GFNI-NEXT:    movabsq $3689348814741910323, %rax # imm = 0x3333333333333333
+; GFNI-NEXT:    movq %rdi, %rcx
+; GFNI-NEXT:    andq %rax, %rcx
+; GFNI-NEXT:    shrq $2, %rdi
+; GFNI-NEXT:    andq %rax, %rdi
+; GFNI-NEXT:    leaq (%rdi,%rcx,4), %rax
+; GFNI-NEXT:    movabsq $6148914691236517205, %rcx # imm = 0x5555555555555555
+; GFNI-NEXT:    movq %rax, %rdx
+; GFNI-NEXT:    andq %rcx, %rdx
+; GFNI-NEXT:    shrq %rax
+; GFNI-NEXT:    andq %rcx, %rax
+; GFNI-NEXT:    leaq (%rax,%rdx,2), %rax
+; GFNI-NEXT:    retq
   %b = call i64 @llvm.bitreverse.i64(i64 %a)
   ret i64 %b
 }
@@ -219,6 +250,28 @@ define i32 @test_bitreverse_i32(i32 %a) nounwind {
 ; X86XOP-NEXT:    vpperm {{\.?LCPI[0-9]+_[0-9]+}}, %xmm0, %xmm0, %xmm0
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: test_bitreverse_i32:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    # kill: def $edi killed $edi def $rdi
+; GFNI-NEXT:    bswapl %edi
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; GFNI-NEXT:    shll $4, %eax
+; GFNI-NEXT:    shrl $4, %edi
+; GFNI-NEXT:    andl $252645135, %edi # imm = 0xF0F0F0F
+; GFNI-NEXT:    orl %eax, %edi
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; GFNI-NEXT:    shrl $2, %edi
+; GFNI-NEXT:    andl $858993459, %edi # imm = 0x33333333
+; GFNI-NEXT:    leal (%rdi,%rax,4), %eax
+; GFNI-NEXT:    movl %eax, %ecx
+; GFNI-NEXT:    andl $1431655765, %ecx # imm = 0x55555555
+; GFNI-NEXT:    shrl %eax
+; GFNI-NEXT:    andl $1431655765, %eax # imm = 0x55555555
+; GFNI-NEXT:    leal (%rax,%rcx,2), %eax
+; GFNI-NEXT:    retq
   %b = call i32 @llvm.bitreverse.i32(i32 %a)
   ret i32 %b
 }
@@ -279,6 +332,29 @@ define i24 @test_bitreverse_i24(i24 %a) nounwind {
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    shrl $8, %eax
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: test_bitreverse_i24:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    # kill: def $edi killed $edi def $rdi
+; GFNI-NEXT:    bswapl %edi
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    andl $252645135, %eax # imm = 0xF0F0F0F
+; GFNI-NEXT:    shll $4, %eax
+; GFNI-NEXT:    shrl $4, %edi
+; GFNI-NEXT:    andl $252645135, %edi # imm = 0xF0F0F0F
+; GFNI-NEXT:    orl %eax, %edi
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    andl $858993459, %eax # imm = 0x33333333
+; GFNI-NEXT:    shrl $2, %edi
+; GFNI-NEXT:    andl $858993459, %edi # imm = 0x33333333
+; GFNI-NEXT:    leal (%rdi,%rax,4), %eax
+; GFNI-NEXT:    movl %eax, %ecx
+; GFNI-NEXT:    andl $1431655680, %ecx # imm = 0x55555500
+; GFNI-NEXT:    shrl %eax
+; GFNI-NEXT:    andl $1431655680, %eax # imm = 0x55555500
+; GFNI-NEXT:    leal (%rax,%rcx,2), %eax
+; GFNI-NEXT:    shrl $8, %eax
+; GFNI-NEXT:    retq
   %b = call i24 @llvm.bitreverse.i24(i24 %a)
   ret i24 %b
 }
@@ -339,6 +415,29 @@ define i16 @test_bitreverse_i16(i16 %a) nounwind {
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    # kill: def $ax killed $ax killed $eax
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: test_bitreverse_i16:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    # kill: def $edi killed $edi def $rdi
+; GFNI-NEXT:    rolw $8, %di
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    andl $3855, %eax # imm = 0xF0F
+; GFNI-NEXT:    shll $4, %eax
+; GFNI-NEXT:    shrl $4, %edi
+; GFNI-NEXT:    andl $3855, %edi # imm = 0xF0F
+; GFNI-NEXT:    orl %eax, %edi
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    andl $13107, %eax # imm = 0x3333
+; GFNI-NEXT:    shrl $2, %edi
+; GFNI-NEXT:    andl $13107, %edi # imm = 0x3333
+; GFNI-NEXT:    leal (%rdi,%rax,4), %eax
+; GFNI-NEXT:    movl %eax, %ecx
+; GFNI-NEXT:    andl $21845, %ecx # imm = 0x5555
+; GFNI-NEXT:    shrl %eax
+; GFNI-NEXT:    andl $21845, %eax # imm = 0x5555
+; GFNI-NEXT:    leal (%rax,%rcx,2), %eax
+; GFNI-NEXT:    # kill: def $ax killed $ax killed $eax
+; GFNI-NEXT:    retq
   %b = call i16 @llvm.bitreverse.i16(i16 %a)
   ret i16 %b
 }
@@ -388,6 +487,23 @@ define i8 @test_bitreverse_i8(i8 %a) {
 ; X86XOP-NEXT:    vmovd %xmm0, %eax
 ; X86XOP-NEXT:    # kill: def $al killed $al killed $eax
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: test_bitreverse_i8:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    rolb $4, %dil
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    andb $51, %al
+; GFNI-NEXT:    shlb $2, %al
+; GFNI-NEXT:    shrb $2, %dil
+; GFNI-NEXT:    andb $51, %dil
+; GFNI-NEXT:    orb %dil, %al
+; GFNI-NEXT:    movl %eax, %ecx
+; GFNI-NEXT:    andb $85, %cl
+; GFNI-NEXT:    addb %cl, %cl
+; GFNI-NEXT:    shrb %al
+; GFNI-NEXT:    andb $85, %al
+; GFNI-NEXT:    orb %cl, %al
+; GFNI-NEXT:    retq
   %b = call i8 @llvm.bitreverse.i8(i8 %a)
   ret i8 %b
 }
@@ -439,6 +555,23 @@ define i4 @test_bitreverse_i4(i4 %a) {
 ; X86XOP-NEXT:    shrb $4, %al
 ; X86XOP-NEXT:    # kill: def $al killed $al killed $eax
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: test_bitreverse_i4:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    # kill: def $edi killed $edi def $rdi
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    andb $8, %al
+; GFNI-NEXT:    leal (%rdi,%rdi), %ecx
+; GFNI-NEXT:    andb $4, %cl
+; GFNI-NEXT:    leal (,%rdi,8), %edx
+; GFNI-NEXT:    andb $8, %dl
+; GFNI-NEXT:    orb %cl, %dl
+; GFNI-NEXT:    shrb %dil
+; GFNI-NEXT:    andb $2, %dil
+; GFNI-NEXT:    orb %dil, %dl
+; GFNI-NEXT:    shrb $3, %al
+; GFNI-NEXT:    orb %dl, %al
+; GFNI-NEXT:    retq
   %b = call i4 @llvm.bitreverse.i4(i4 %a)
   ret i4 %b
 }
@@ -461,63 +594,38 @@ define <2 x i16> @fold_v2i16() {
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    vmovaps {{.*#+}} xmm0 = <61440,240,u,u,u,u,u,u>
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: fold_v2i16:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    vmovaps {{.*#+}} xmm0 = <61440,240,u,u,u,u,u,u>
+; GFNI-NEXT:    retq
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> <i16 15, i16 3840>)
   ret <2 x i16> %b
 }
 
 define i24 @fold_i24() {
-; X86-LABEL: fold_i24:
-; X86:       # %bb.0:
-; X86-NEXT:    movl $2048, %eax # imm = 0x800
-; X86-NEXT:    retl
-;
-; X64-LABEL: fold_i24:
-; X64:       # %bb.0:
-; X64-NEXT:    movl $2048, %eax # imm = 0x800
-; X64-NEXT:    retq
-;
-; X86XOP-LABEL: fold_i24:
-; X86XOP:       # %bb.0:
-; X86XOP-NEXT:    movl $2048, %eax # imm = 0x800
-; X86XOP-NEXT:    retl
+; CHECK-LABEL: fold_i24:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movl $2048, %eax # imm = 0x800
+; CHECK-NEXT:    ret{{[l|q]}}
   %b = call i24 @llvm.bitreverse.i24(i24 4096)
   ret i24 %b
 }
 
 define i8 @fold_i8() {
-; X86-LABEL: fold_i8:
-; X86:       # %bb.0:
-; X86-NEXT:    movb $-16, %al
-; X86-NEXT:    retl
-;
-; X64-LABEL: fold_i8:
-; X64:       # %bb.0:
-; X64-NEXT:    movb $-16, %al
-; X64-NEXT:    retq
-;
-; X86XOP-LABEL: fold_i8:
-; X86XOP:       # %bb.0:
-; X86XOP-NEXT:    movb $-16, %al
-; X86XOP-NEXT:    retl
+; CHECK-LABEL: fold_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb $-16, %al
+; CHECK-NEXT:    ret{{[l|q]}}
   %b = call i8 @llvm.bitreverse.i8(i8 15)
   ret i8 %b
 }
 
 define i4 @fold_i4() {
-; X86-LABEL: fold_i4:
-; X86:       # %bb.0:
-; X86-NEXT:    movb $1, %al
-; X86-NEXT:    retl
-;
-; X64-LABEL: fold_i4:
-; X64:       # %bb.0:
-; X64-NEXT:    movb $1, %al
-; X64-NEXT:    retq
-;
-; X86XOP-LABEL: fold_i4:
-; X86XOP:       # %bb.0:
-; X86XOP-NEXT:    movb $1, %al
-; X86XOP-NEXT:    retl
+; CHECK-LABEL: fold_i4:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    movb $1, %al
+; CHECK-NEXT:    ret{{[l|q]}}
   %b = call i4 @llvm.bitreverse.i4(i4 8)
   ret i4 %b
 }
@@ -540,6 +648,12 @@ define i8 @identity_i8(i8 %a) {
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    movzbl {{[0-9]+}}(%esp), %eax
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: identity_i8:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    movl %edi, %eax
+; GFNI-NEXT:    # kill: def $al killed $al killed $eax
+; GFNI-NEXT:    retq
   %b = call i8 @llvm.bitreverse.i8(i8 %a)
   %c = call i8 @llvm.bitreverse.i8(i8 %b)
   ret i8 %c
@@ -559,6 +673,10 @@ define <2 x i16> @identity_v2i16(<2 x i16> %a) {
 ; X86XOP-LABEL: identity_v2i16:
 ; X86XOP:       # %bb.0:
 ; X86XOP-NEXT:    retl
+;
+; GFNI-LABEL: identity_v2i16:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    retq
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %a)
   %c = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> %b)
   ret <2 x i16> %c
@@ -567,33 +685,17 @@ define <2 x i16> @identity_v2i16(<2 x i16> %a) {
 ; These tests check that bitreverse(undef) calls are removed
 
 define i8 @undef_i8() {
-; X86-LABEL: undef_i8:
-; X86:       # %bb.0:
-; X86-NEXT:    retl
-;
-; X64-LABEL: undef_i8:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
-;
-; X86XOP-LABEL: undef_i8:
-; X86XOP:       # %bb.0:
-; X86XOP-NEXT:    retl
+; CHECK-LABEL: undef_i8:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %b = call i8 @llvm.bitreverse.i8(i8 undef)
   ret i8 %b
 }
 
 define <2 x i16> @undef_v2i16() {
-; X86-LABEL: undef_v2i16:
-; X86:       # %bb.0:
-; X86-NEXT:    retl
-;
-; X64-LABEL: undef_v2i16:
-; X64:       # %bb.0:
-; X64-NEXT:    retq
-;
-; X86XOP-LABEL: undef_v2i16:
-; X86XOP:       # %bb.0:
-; X86XOP-NEXT:    retl
+; CHECK-LABEL: undef_v2i16:
+; CHECK:       # %bb.0:
+; CHECK-NEXT:    ret{{[l|q]}}
   %b = call <2 x i16> @llvm.bitreverse.v2i16(<2 x i16> undef)
   ret <2 x i16> %b
 }
@@ -1310,6 +1412,202 @@ define i528 @large_promotion(i528 %A) nounwind {
 ; X86XOP-NEXT:    popl %ebx
 ; X86XOP-NEXT:    popl %ebp
 ; X86XOP-NEXT:    retl $4
+;
+; GFNI-LABEL: large_promotion:
+; GFNI:       # %bb.0:
+; GFNI-NEXT:    pushq %rbp
+; GFNI-NEXT:    pushq %r15
+; GFNI-NEXT:    pushq %r14
+; GFNI-NEXT:    pushq %r13
+; GFNI-NEXT:    pushq %r12
+; GFNI-NEXT:    pushq %rbx
+; GFNI-NEXT:    movq %rdi, %rax
+; GFNI-NEXT:    movq {{[0-9]+}}(%rsp), %r12
+; GFNI-NEXT:    movq {{[0-9]+}}(%rsp), %r15
+; GFNI-NEXT:    movq {{[0-9]+}}(%rsp), %rbp
+; GFNI-NEXT:    movq {{[0-9]+}}(%rsp), %rdi
+; GFNI-NEXT:    bswapq %rdi
+; GFNI-NEXT:    movq %rdi, %rbx
+; GFNI-NEXT:    shrq $4, %rbx
+; GFNI-NEXT:    movabsq $1085102592571150095, %r13 # imm = 0xF0F0F0F0F0F0F0F
+; GFNI-NEXT:    andq %r13, %rbx
+; GFNI-NEXT:    andq %r13, %rdi
+; GFNI-NEXT:    shlq $4, %rdi
+; GFNI-NEXT:    orq %rbx, %rdi
+; GFNI-NEXT:    movabsq $3689348814741910323, %r11 # imm = 0x3333333333333333
+; GFNI-NEXT:    movq %rdi, %rbx
+; GFNI-NEXT:    andq %r11, %rbx
+; GFNI-NEXT:    shrq $2, %rdi
+; GFNI-NEXT:    andq %r11, %rdi
+; GFNI-NEXT:    leaq (%rdi,%rbx,4), %rdi
+; GFNI-NEXT:    movabsq $6148820866244280320, %r10 # imm = 0x5555000000000000
+; GFNI-NEXT:    movq %rdi, %rbx
+; GFNI-NEXT:    andq %r10, %rbx
+; GFNI-NEXT:    shrq %rdi
+; GFNI-NEXT:    andq %r10, %rdi
+; GFNI-NEXT:    leaq (%rdi,%rbx,2), %r10
+; GFNI-NEXT:    bswapq %rbp
+; GFNI-NEXT:    movq %rbp, %rdi
+; GFNI-NEXT:    shrq $4, %rdi
+; GFNI-NEXT:    andq %r13, %rdi
+; GFNI-NEXT:    andq %r13, %rbp
+; GFNI-NEXT:    shlq $4, %rbp
+; GFNI-NEXT:    orq %rdi, %rbp
+; GFNI-NEXT:    movq %rbp, %rdi
+; GFNI-NEXT:    andq %r11, %rdi
+; GFNI-NEXT:    shrq $2, %rbp
+; GFNI-NEXT:    andq %r11, %rbp
+; GFNI-NEXT:    leaq (%rbp,%rdi,4), %rdi
+; GFNI-NEXT:    movabsq $6148914691236517205, %rbp # imm = 0x5555555555555555
+; GFNI-NEXT:    movq %rdi, %rbx
+; GFNI-NEXT:    andq %rbp, %rbx
+; GFNI-NEXT:    shrq %rdi
+; GFNI-NEXT:    andq %rbp, %rdi
+; GFNI-NEXT:    leaq (%rdi,%rbx,2), %r14
+; GFNI-NEXT:    shrdq $48, %r14, %r10
+; GFNI-NEXT:    bswapq %r15
+; GFNI-NEXT:    movq %r15, %rdi
+; GFNI-NEXT:    shrq $4, %rdi
+; GFNI-NEXT:    andq %r13, %rdi
+; GFNI-NEXT:    andq %r13, %r15
+; GFNI-NEXT:    shlq $4, %r15
+; GFNI-NEXT:    orq %rdi, %r15
+; GFNI-NEXT:    movq %r15, %rdi
+; GFNI-NEXT:    andq %r11, %rdi
+; GFNI-NEXT:    shrq $2, %r15
+; GFNI-NEXT:    andq %r11, %r15
+; GFNI-NEXT:    leaq (%r15,%rdi,4), %rdi
+; GFNI-NEXT:    movq %rdi, %rbx
+; GFNI-NEXT:    andq %rbp, %rbx
+; GFNI-NEXT:    shrq %rdi
+; GFNI-NEXT:    andq %rbp, %rdi
+; GFNI-NEXT:    leaq (%rdi,%rbx,2), %r15
+; GFNI-NEXT:    shrdq $48, %r15, %r14
+; GFNI-NEXT:    bswapq %r12
+; GFNI-NEXT:    movq %r12, %rdi
+; GFNI-NEXT:    shrq $4, %rdi
+; GFNI-NEXT:    andq %r13, %rdi
+; GFNI-NEXT:    andq %r13, %r12
+; GFNI-NEXT:    shlq $4, %r12
+; GFNI-NEXT:    orq %rdi, %r12
+; GFNI-NEXT:    movq %r12, %rdi
+; GFNI-NEXT:    andq %r11, %rdi
+; GFNI-NEXT:    shrq $2, %r12
+; GFNI-NEXT:    andq %r11, %r12
+; GFNI-NEXT:    leaq (%r12,%rdi,4), %rdi
+; GFNI-NEXT:    movq %rdi, %rbx
+; GFNI-NEXT:    andq %rbp, %rbx
+; GFNI-NEXT:    shrq %rdi
+; GFNI-NEXT:    andq %rbp, %rdi
+; GFNI-NEXT:    leaq (%rdi,%rbx,2), %r12
+; GFNI-NEXT:    shrdq $48, %r12, %r15
+; GFNI-NEXT:    bswapq %r9
+; GFNI-NEXT:    movq %r9, %rdi
+; GFNI-NEXT:    shrq $4, %rdi
+; GFNI-NEXT:    andq %r13, %rdi
+; GFNI-NEXT:    andq %r13, %r9
+; GFNI-NEXT:    shlq $4, %r9
+; GFNI-NEXT:    orq %rdi, %r9
+; GFNI-NEXT:    movq %r9, %rdi
+; GFNI-NEXT:    andq %r11, %rdi
+; GFNI-NEXT:    shrq $2, %r9
+; GFNI-NEXT:    andq %r11, %r9
+; GFNI-NEXT:    leaq (%r9,%rdi,4), %rdi
+; GFNI-NEXT:    movq %rdi, %rbx
+; GFNI-NEXT:    andq %rbp, %rbx
+; GFNI-NEXT:    shrq %rdi
+; GFNI-NEXT:    andq %rbp, %rdi
+; GFNI-NEXT:    leaq (%rdi,%rbx,2), %r9
+; GFNI-NEXT:    shrdq $48, %r9, %r12
+; GFNI-NEXT:    bswapq %r8
+; GFNI-NEXT:    movq %r8, %rdi
+; GFNI-NEXT:    shrq $4, %rdi
+; GFNI-NEXT:    andq %r13, %rdi
+; GFNI-NEXT:    andq %r13, %r8
+; GFNI-NEXT:    shlq $4, %r8
+; GFNI-NEXT:    orq %rdi, %r8
+; GFNI-NEXT:    movq %r8, %rdi
+; GFNI-NEXT:    andq %r11, %rdi
+; GFNI-NEXT:    shrq $2, %r8
+; GFNI-NEXT:    andq %r11, %r8
+; GFNI-NEXT:    leaq (%r8,%rdi,4), %rdi
+; GFNI-NEXT:    movq %rdi, %rbx
+; GFNI-NEXT:    andq %rbp, %rbx
+; GFNI-NEXT:    shrq %rdi
+; GFNI-NEXT:    andq %rbp, %rdi
+; GFNI-NEXT:    leaq (%rdi,%rbx,2), %rdi
+; GFNI-NEXT:    shrdq $48, %rdi, %r9
+; GFNI-NEXT:    bswapq %rcx
+; GFNI-NEXT:    movq %rcx, %rbx
+; GFNI-NEXT:    shrq $4, %rbx
+; GFNI-NEXT:    andq %r13, %rbx
+; GFNI-NEXT:    andq %r13, %rcx
+; GFNI-NEXT:    shlq $4, %rcx
+; GFNI-NEXT:    orq %rbx, %rcx
+; GFNI-NEXT:    movq %rcx, %rbx
+; GFNI-NEXT:    andq %r11, %rbx
+; GFNI-NEXT:    shrq $2, %rcx
+; GFNI-NEXT:    andq %r11, %rcx
+; GFNI-NEXT:    leaq (%rcx,%rbx,4), %rcx
+; GFNI-NEXT:    movq %rcx, %rbx
+; GFNI-NEXT:    andq %rbp, %rbx
+; GFNI-NEXT:    shrq %rcx
+; GFNI-NEXT:    andq %rbp, %rcx
+; GFNI-NEXT:    leaq (%rcx,%rbx,2), %rcx
+; GFNI-NEXT:    shrdq $48, %rcx, %rdi
+; GFNI-NEXT:    bswapq %rdx
+; GFNI-NEXT:    movq %rdx, %rbx
+; GFNI-NEXT:    shrq $4, %rbx
+; GFNI-NEXT:    andq %r13, %rbx
+; GFNI-NEXT:    andq %r13, %rdx
+; GFNI-NEXT:    shlq $4, %rdx
+; GFNI-NEXT:    orq %rbx, %rdx
+; GFNI-NEXT:    movq %rdx, %rbx
+; GFNI-NEXT:    andq %r11, %rbx
+; GFNI-NEXT:    shrq $2, %rdx
+; GFNI-NEXT:    andq %r11, %rdx
+; GFNI-NEXT:    leaq (%rdx,%rbx,4), %rdx
+; GFNI-NEXT:    movq %rdx, %rbx
+; GFNI-NEXT:    andq %rbp, %rbx
+; GFNI-NEXT:    shrq %rdx
+; GFNI-NEXT:    andq %rbp, %rdx
+; GFNI-NEXT:    leaq (%rdx,%rbx,2), %rdx
+; GFNI-NEXT:    shrdq $48, %rdx, %rcx
+; GFNI-NEXT:    bswapq %rsi
+; GFNI-NEXT:    movq %rsi, %rbx
+; GFNI-NEXT:    shrq $4, %rbx
+; GFNI-NEXT:    andq %r13, %rbx
+; GFNI-NEXT:    andq %r13, %rsi
+; GFNI-NEXT:    shlq $4, %rsi
+; GFNI-NEXT:    orq %rbx, %rsi
+; GFNI-NEXT:    movq %rsi, %rbx
+; GFNI-NEXT:    andq %r11, %rbx
+; GFNI-NEXT:    shrq $2, %rsi
+; GFNI-NEXT:    andq %r11, %rsi
+; GFNI-NEXT:    leaq (%rsi,%rbx,4), %rsi
+; GFNI-NEXT:    movq %rsi, %rbx
+; GFNI-NEXT:    andq %rbp, %rbx
+; GFNI-NEXT:    shrq %rsi
+; GFNI-NEXT:    andq %rbp, %rsi
+; GFNI-NEXT:    leaq (%rsi,%rbx,2), %rsi
+; GFNI-NEXT:    shrdq $48, %rsi, %rdx
+; GFNI-NEXT:    shrq $48, %rsi
+; GFNI-NEXT:    movq %rdx, 56(%rax)
+; GFNI-NEXT:    movq %rcx, 48(%rax)
+; GFNI-NEXT:    movq %rdi, 40(%rax)
+; GFNI-NEXT:    movq %r9, 32(%rax)
+; GFNI-NEXT:    movq %r12, 24(%rax)
+; GFNI-NEXT:    movq %r15, 16(%rax)
+; GFNI-NEXT:    movq %r14, 8(%rax)
+; GFNI-NEXT:    movq %r10, (%rax)
+; GFNI-NEXT:    movw %si, 64(%rax)
+; GFNI-NEXT:    popq %rbx
+; GFNI-NEXT:    popq %r12
+; GFNI-NEXT:    popq %r13
+; GFNI-NEXT:    popq %r14
+; GFNI-NEXT:    popq %r15
+; GFNI-NEXT:    popq %rbp
+; GFNI-NEXT:    retq
   %Z = call i528 @llvm.bitreverse.i528(i528 %A)
   ret i528 %Z
 }