b563e76306
When available, we can use SVE's CNT instruction to improve the lowering of scalar and fixed-length popcount (CTPOP) since the SVE instruction supports types that the Neon variant doesn't. For the scalar types, I see the following speedups on NVIDIA Grace CPU: | size (bits) | before (Gibit/s) | after (Gibit/s) | speedup | |------------:|-----------------:|----------------:|--------:| | 32 | 75.20 | 86.79 | 1.15 | | 64 | 149.87 | 173.70 | 1.16 | | 128 | 158.56 | 164.88 | 1.04 |
907 lines
28 KiB
LLVM
907 lines
28 KiB
LLVM
; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
|
|
; RUN: llc < %s -O0 -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=CHECKO0
|
|
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=CHECK,NEON
|
|
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=CHECK,DOT
|
|
; RUN: llc < %s -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=CHECK,SVE
|
|
; RUN: llc < %s -mtriple=aarch64_be-unknown-unknown | FileCheck %s --check-prefixes=BE
|
|
; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=GISEL
|
|
; RUN: llc < %s -O0 -global-isel -mtriple=aarch64-unknown-unknown | FileCheck %s --check-prefix=GISELO0
|
|
; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+neon | FileCheck %s --check-prefixes=GISEL,NEON-GISEL
|
|
; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+neon,+dotprod | FileCheck %s --check-prefixes=GISEL,DOT-GISEL
|
|
; RUN: llc < %s -global-isel -mtriple=aarch64-unknown-unknown -mattr=+sve | FileCheck %s --check-prefixes=GISEL,SVE-GISEL
|
|
|
|
|
|
; Function Attrs: nobuiltin nounwind readonly
|
|
define i8 @popcount128(ptr nocapture nonnull readonly %0) {
|
|
; CHECKO0-LABEL: popcount128:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: ldr q0, [x0]
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlv h0, v0.16b
|
|
; CHECKO0-NEXT: // kill: def $q0 killed $h0
|
|
; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; CHECKO0-NEXT: fmov w0, s0
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount128:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: ldr d0, [x0]
|
|
; NEON-NEXT: add x8, x0, #8
|
|
; NEON-NEXT: ld1 { v0.d }[1], [x8]
|
|
; NEON-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-NEXT: addv b0, v0.16b
|
|
; NEON-NEXT: fmov w0, s0
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount128:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: ldr d0, [x0]
|
|
; DOT-NEXT: add x8, x0, #8
|
|
; DOT-NEXT: ld1 { v0.d }[1], [x8]
|
|
; DOT-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-NEXT: addv b0, v0.16b
|
|
; DOT-NEXT: fmov w0, s0
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount128:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: ldr d0, [x0]
|
|
; SVE-NEXT: add x8, x0, #8
|
|
; SVE-NEXT: ptrue p0.d
|
|
; SVE-NEXT: ld1 { v0.d }[1], [x8]
|
|
; SVE-NEXT: cnt z0.d, p0/m, z0.d
|
|
; SVE-NEXT: addp d0, v0.2d
|
|
; SVE-NEXT: fmov w0, s0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount128:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: ldr d0, [x0]
|
|
; BE-NEXT: add x8, x0, #8
|
|
; BE-NEXT: ld1 { v0.d }[1], [x8]
|
|
; BE-NEXT: rev64 v0.16b, v0.16b
|
|
; BE-NEXT: cnt v0.16b, v0.16b
|
|
; BE-NEXT: addv b0, v0.16b
|
|
; BE-NEXT: rev64 v0.4s, v0.4s
|
|
; BE-NEXT: mov w0, v0.s[1]
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount128:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: ldr q0, [x0]
|
|
; GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; GISEL-NEXT: uaddlv h0, v0.16b
|
|
; GISEL-NEXT: fmov w0, s0
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount128:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: ldr q0, [x0]
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlv h0, v0.16b
|
|
; GISELO0-NEXT: // kill: def $q0 killed $h0
|
|
; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; GISELO0-NEXT: fmov w0, s0
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = load i128, ptr %0, align 16
|
|
%2 = tail call i128 @llvm.ctpop.i128(i128 %1)
|
|
%3 = trunc i128 %2 to i8
|
|
ret i8 %3
|
|
}
|
|
|
|
; Function Attrs: nounwind readnone speculatable willreturn
|
|
declare i128 @llvm.ctpop.i128(i128)
|
|
|
|
; Function Attrs: nobuiltin nounwind readonly
|
|
define i16 @popcount256(ptr nocapture nonnull readonly %0) {
|
|
; CHECKO0-LABEL: popcount256:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: ldr x11, [x0]
|
|
; CHECKO0-NEXT: ldr x10, [x0, #8]
|
|
; CHECKO0-NEXT: ldr x9, [x0, #16]
|
|
; CHECKO0-NEXT: ldr x8, [x0, #24]
|
|
; CHECKO0-NEXT: // implicit-def: $q1
|
|
; CHECKO0-NEXT: mov v1.d[0], x11
|
|
; CHECKO0-NEXT: mov v1.d[1], x10
|
|
; CHECKO0-NEXT: // implicit-def: $q0
|
|
; CHECKO0-NEXT: mov v0.d[0], x9
|
|
; CHECKO0-NEXT: mov v0.d[1], x8
|
|
; CHECKO0-NEXT: cnt v1.16b, v1.16b
|
|
; CHECKO0-NEXT: uaddlv h1, v1.16b
|
|
; CHECKO0-NEXT: // kill: def $q1 killed $h1
|
|
; CHECKO0-NEXT: // kill: def $s1 killed $s1 killed $q1
|
|
; CHECKO0-NEXT: fmov w0, s1
|
|
; CHECKO0-NEXT: mov w10, wzr
|
|
; CHECKO0-NEXT: mov w9, w0
|
|
; CHECKO0-NEXT: mov w8, w10
|
|
; CHECKO0-NEXT: bfi x9, x8, #32, #32
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlv h0, v0.16b
|
|
; CHECKO0-NEXT: // kill: def $q0 killed $h0
|
|
; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; CHECKO0-NEXT: fmov w0, s0
|
|
; CHECKO0-NEXT: mov w8, w0
|
|
; CHECKO0-NEXT: // kill: def $x10 killed $w10
|
|
; CHECKO0-NEXT: bfi x8, x10, #32, #32
|
|
; CHECKO0-NEXT: adds x8, x8, x9
|
|
; CHECKO0-NEXT: mov w0, w8
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount256:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: ldr d0, [x0, #16]
|
|
; NEON-NEXT: ldr d1, [x0]
|
|
; NEON-NEXT: add x8, x0, #8
|
|
; NEON-NEXT: add x9, x0, #24
|
|
; NEON-NEXT: ld1 { v0.d }[1], [x9]
|
|
; NEON-NEXT: ld1 { v1.d }[1], [x8]
|
|
; NEON-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-NEXT: cnt v1.16b, v1.16b
|
|
; NEON-NEXT: addv b0, v0.16b
|
|
; NEON-NEXT: addv b1, v1.16b
|
|
; NEON-NEXT: fmov w8, s0
|
|
; NEON-NEXT: fmov w9, s1
|
|
; NEON-NEXT: add w0, w9, w8
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount256:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: ldr d0, [x0, #16]
|
|
; DOT-NEXT: ldr d1, [x0]
|
|
; DOT-NEXT: add x8, x0, #8
|
|
; DOT-NEXT: add x9, x0, #24
|
|
; DOT-NEXT: ld1 { v0.d }[1], [x9]
|
|
; DOT-NEXT: ld1 { v1.d }[1], [x8]
|
|
; DOT-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-NEXT: cnt v1.16b, v1.16b
|
|
; DOT-NEXT: addv b0, v0.16b
|
|
; DOT-NEXT: addv b1, v1.16b
|
|
; DOT-NEXT: fmov w8, s0
|
|
; DOT-NEXT: fmov w9, s1
|
|
; DOT-NEXT: add w0, w9, w8
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount256:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: ldr d0, [x0, #16]
|
|
; SVE-NEXT: ldr d1, [x0]
|
|
; SVE-NEXT: add x8, x0, #8
|
|
; SVE-NEXT: add x9, x0, #24
|
|
; SVE-NEXT: ptrue p0.d
|
|
; SVE-NEXT: ld1 { v0.d }[1], [x9]
|
|
; SVE-NEXT: ld1 { v1.d }[1], [x8]
|
|
; SVE-NEXT: cnt z0.d, p0/m, z0.d
|
|
; SVE-NEXT: cnt z1.d, p0/m, z1.d
|
|
; SVE-NEXT: add v0.2d, v1.2d, v0.2d
|
|
; SVE-NEXT: addp d0, v0.2d
|
|
; SVE-NEXT: fmov w0, s0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount256:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: ldr d0, [x0]
|
|
; BE-NEXT: ldr d1, [x0, #16]
|
|
; BE-NEXT: add x8, x0, #24
|
|
; BE-NEXT: add x9, x0, #8
|
|
; BE-NEXT: ld1 { v0.d }[1], [x9]
|
|
; BE-NEXT: ld1 { v1.d }[1], [x8]
|
|
; BE-NEXT: rev64 v0.16b, v0.16b
|
|
; BE-NEXT: rev64 v1.16b, v1.16b
|
|
; BE-NEXT: cnt v0.16b, v0.16b
|
|
; BE-NEXT: cnt v1.16b, v1.16b
|
|
; BE-NEXT: addv b0, v0.16b
|
|
; BE-NEXT: addv b1, v1.16b
|
|
; BE-NEXT: rev64 v0.4s, v0.4s
|
|
; BE-NEXT: rev64 v1.4s, v1.4s
|
|
; BE-NEXT: mov w8, v0.s[1]
|
|
; BE-NEXT: mov w9, v1.s[1]
|
|
; BE-NEXT: add w0, w9, w8
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount256:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: ldp x8, x9, [x0]
|
|
; GISEL-NEXT: mov v0.d[0], x8
|
|
; GISEL-NEXT: ldp x8, x10, [x0, #16]
|
|
; GISEL-NEXT: mov v1.d[0], x8
|
|
; GISEL-NEXT: mov v0.d[1], x9
|
|
; GISEL-NEXT: mov v1.d[1], x10
|
|
; GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; GISEL-NEXT: cnt v1.16b, v1.16b
|
|
; GISEL-NEXT: uaddlv h0, v0.16b
|
|
; GISEL-NEXT: uaddlv h1, v1.16b
|
|
; GISEL-NEXT: mov w8, v0.s[0]
|
|
; GISEL-NEXT: fmov w9, s1
|
|
; GISEL-NEXT: add x0, x8, w9, uxtw
|
|
; GISEL-NEXT: // kill: def $w0 killed $w0 killed $x0
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount256:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: ldr x11, [x0]
|
|
; GISELO0-NEXT: ldr x10, [x0, #8]
|
|
; GISELO0-NEXT: ldr x9, [x0, #16]
|
|
; GISELO0-NEXT: ldr x8, [x0, #24]
|
|
; GISELO0-NEXT: // implicit-def: $q1
|
|
; GISELO0-NEXT: mov v1.d[0], x11
|
|
; GISELO0-NEXT: mov v1.d[1], x10
|
|
; GISELO0-NEXT: // implicit-def: $q0
|
|
; GISELO0-NEXT: mov v0.d[0], x9
|
|
; GISELO0-NEXT: mov v0.d[1], x8
|
|
; GISELO0-NEXT: cnt v1.16b, v1.16b
|
|
; GISELO0-NEXT: uaddlv h1, v1.16b
|
|
; GISELO0-NEXT: // kill: def $q1 killed $h1
|
|
; GISELO0-NEXT: // kill: def $s1 killed $s1 killed $q1
|
|
; GISELO0-NEXT: fmov w0, s1
|
|
; GISELO0-NEXT: mov w10, wzr
|
|
; GISELO0-NEXT: mov w9, w0
|
|
; GISELO0-NEXT: mov w8, w10
|
|
; GISELO0-NEXT: bfi x9, x8, #32, #32
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlv h0, v0.16b
|
|
; GISELO0-NEXT: // kill: def $q0 killed $h0
|
|
; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; GISELO0-NEXT: fmov w0, s0
|
|
; GISELO0-NEXT: mov w8, w0
|
|
; GISELO0-NEXT: // kill: def $x10 killed $w10
|
|
; GISELO0-NEXT: bfi x8, x10, #32, #32
|
|
; GISELO0-NEXT: adds x8, x8, x9
|
|
; GISELO0-NEXT: mov w0, w8
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = load i256, ptr %0, align 16
|
|
%2 = tail call i256 @llvm.ctpop.i256(i256 %1)
|
|
%3 = trunc i256 %2 to i16
|
|
ret i16 %3
|
|
}
|
|
|
|
; Function Attrs: nounwind readnone speculatable willreturn
|
|
declare i256 @llvm.ctpop.i256(i256)
|
|
|
|
define <1 x i128> @popcount1x128(<1 x i128> %0) {
|
|
; CHECKO0-LABEL: popcount1x128:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: // implicit-def: $q0
|
|
; CHECKO0-NEXT: mov v0.d[0], x0
|
|
; CHECKO0-NEXT: mov v0.d[1], x1
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlv h0, v0.16b
|
|
; CHECKO0-NEXT: // kill: def $q0 killed $h0
|
|
; CHECKO0-NEXT: mov x1, xzr
|
|
; CHECKO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; CHECKO0-NEXT: fmov w0, s0
|
|
; CHECKO0-NEXT: mov w8, wzr
|
|
; CHECKO0-NEXT: // kill: def $x0 killed $w0
|
|
; CHECKO0-NEXT: // kill: def $x8 killed $w8
|
|
; CHECKO0-NEXT: bfi x0, x8, #32, #32
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount1x128:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: fmov d0, x0
|
|
; NEON-NEXT: mov v0.d[1], x1
|
|
; NEON-NEXT: mov x1, xzr
|
|
; NEON-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-NEXT: addv b0, v0.16b
|
|
; NEON-NEXT: fmov x0, d0
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount1x128:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: fmov d0, x0
|
|
; DOT-NEXT: mov v0.d[1], x1
|
|
; DOT-NEXT: mov x1, xzr
|
|
; DOT-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-NEXT: addv b0, v0.16b
|
|
; DOT-NEXT: fmov x0, d0
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount1x128:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: fmov d0, x0
|
|
; SVE-NEXT: ptrue p0.d
|
|
; SVE-NEXT: mov v0.d[1], x1
|
|
; SVE-NEXT: mov x1, xzr
|
|
; SVE-NEXT: cnt z0.d, p0/m, z0.d
|
|
; SVE-NEXT: addp d0, v0.2d
|
|
; SVE-NEXT: fmov x0, d0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount1x128:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: fmov d0, x0
|
|
; BE-NEXT: mov x0, xzr
|
|
; BE-NEXT: mov v0.d[1], x1
|
|
; BE-NEXT: rev64 v0.16b, v0.16b
|
|
; BE-NEXT: cnt v0.16b, v0.16b
|
|
; BE-NEXT: addv b0, v0.16b
|
|
; BE-NEXT: fmov x1, d0
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount1x128:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: mov v0.d[0], x0
|
|
; GISEL-NEXT: mov v0.d[1], x1
|
|
; GISEL-NEXT: mov x1, xzr
|
|
; GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; GISEL-NEXT: uaddlv h0, v0.16b
|
|
; GISEL-NEXT: mov w0, v0.s[0]
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount1x128:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: // implicit-def: $q0
|
|
; GISELO0-NEXT: mov v0.d[0], x0
|
|
; GISELO0-NEXT: mov v0.d[1], x1
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlv h0, v0.16b
|
|
; GISELO0-NEXT: // kill: def $q0 killed $h0
|
|
; GISELO0-NEXT: mov x1, xzr
|
|
; GISELO0-NEXT: // kill: def $s0 killed $s0 killed $q0
|
|
; GISELO0-NEXT: fmov w0, s0
|
|
; GISELO0-NEXT: mov w8, wzr
|
|
; GISELO0-NEXT: // kill: def $x0 killed $w0
|
|
; GISELO0-NEXT: // kill: def $x8 killed $w8
|
|
; GISELO0-NEXT: bfi x0, x8, #32, #32
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <1 x i128> @llvm.ctpop.v1i128(<1 x i128> %0)
|
|
ret <1 x i128> %1
|
|
}
|
|
|
|
declare <1 x i128> @llvm.ctpop.v1i128(<1 x i128>)
|
|
|
|
define <2 x i64> @popcount2x64(<2 x i64> %0) {
|
|
; CHECKO0-LABEL: popcount2x64:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.4s, v0.8h
|
|
; CHECKO0-NEXT: uaddlp v0.2d, v0.4s
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount2x64:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-NEXT: uaddlp v0.4s, v0.8h
|
|
; NEON-NEXT: uaddlp v0.2d, v0.4s
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount2x64:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: movi v1.16b, #1
|
|
; DOT-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-NEXT: movi v2.2d, #0000000000000000
|
|
; DOT-NEXT: udot v2.4s, v1.16b, v0.16b
|
|
; DOT-NEXT: uaddlp v0.2d, v2.4s
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount2x64:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: ptrue p0.d, vl2
|
|
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
|
|
; SVE-NEXT: cnt z0.d, p0/m, z0.d
|
|
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount2x64:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: rev64 v0.16b, v0.16b
|
|
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
|
|
; BE-NEXT: cnt v0.16b, v0.16b
|
|
; BE-NEXT: uaddlp v0.8h, v0.16b
|
|
; BE-NEXT: uaddlp v0.4s, v0.8h
|
|
; BE-NEXT: uaddlp v0.2d, v0.4s
|
|
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount2x64:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.4s, v0.8h
|
|
; GISELO0-NEXT: uaddlp v0.2d, v0.4s
|
|
; GISELO0-NEXT: ret
|
|
;
|
|
; NEON-GISEL-LABEL: popcount2x64:
|
|
; NEON-GISEL: // %bb.0: // %Entry
|
|
; NEON-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-GISEL-NEXT: uaddlp v0.4s, v0.8h
|
|
; NEON-GISEL-NEXT: uaddlp v0.2d, v0.4s
|
|
; NEON-GISEL-NEXT: ret
|
|
;
|
|
; DOT-GISEL-LABEL: popcount2x64:
|
|
; DOT-GISEL: // %bb.0: // %Entry
|
|
; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
|
|
; DOT-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-GISEL-NEXT: movi v2.16b, #1
|
|
; DOT-GISEL-NEXT: udot v1.4s, v2.16b, v0.16b
|
|
; DOT-GISEL-NEXT: uaddlp v0.2d, v1.4s
|
|
; DOT-GISEL-NEXT: ret
|
|
;
|
|
; SVE-GISEL-LABEL: popcount2x64:
|
|
; SVE-GISEL: // %bb.0: // %Entry
|
|
; SVE-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h
|
|
; SVE-GISEL-NEXT: uaddlp v0.2d, v0.4s
|
|
; SVE-GISEL-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %0)
|
|
ret <2 x i64> %1
|
|
}
|
|
|
|
declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>)
|
|
|
|
define <1 x i64> @popcount1x64(<1 x i64> %0) {
|
|
; CHECKO0-LABEL: popcount1x64:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: fmov x0, d0
|
|
; CHECKO0-NEXT: fmov d0, x0
|
|
; CHECKO0-NEXT: cnt v0.8b, v0.8b
|
|
; CHECKO0-NEXT: uaddlv h0, v0.8b
|
|
; CHECKO0-NEXT: // kill: def $q0 killed $h0
|
|
; CHECKO0-NEXT: mov w8, v0.s[0]
|
|
; CHECKO0-NEXT: // kill: def $x8 killed $w8
|
|
; CHECKO0-NEXT: fmov d0, x8
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount1x64:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.8b, v0.8b
|
|
; NEON-NEXT: uaddlp v0.4h, v0.8b
|
|
; NEON-NEXT: uaddlp v0.2s, v0.4h
|
|
; NEON-NEXT: uaddlp v0.1d, v0.2s
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount1x64:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: cnt v0.8b, v0.8b
|
|
; DOT-NEXT: uaddlp v0.4h, v0.8b
|
|
; DOT-NEXT: uaddlp v0.2s, v0.4h
|
|
; DOT-NEXT: uaddlp v0.1d, v0.2s
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount1x64:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: ptrue p0.d, vl1
|
|
; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
|
|
; SVE-NEXT: cnt z0.d, p0/m, z0.d
|
|
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount1x64:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: rev64 v0.8b, v0.8b
|
|
; BE-NEXT: cnt v0.8b, v0.8b
|
|
; BE-NEXT: uaddlp v0.4h, v0.8b
|
|
; BE-NEXT: uaddlp v0.2s, v0.4h
|
|
; BE-NEXT: uaddlp v0.1d, v0.2s
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount1x64:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; GISEL-NEXT: uaddlv h0, v0.8b
|
|
; GISEL-NEXT: mov w8, v0.s[0]
|
|
; GISEL-NEXT: fmov d0, x8
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount1x64:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: fmov x0, d0
|
|
; GISELO0-NEXT: fmov d0, x0
|
|
; GISELO0-NEXT: cnt v0.8b, v0.8b
|
|
; GISELO0-NEXT: uaddlv h0, v0.8b
|
|
; GISELO0-NEXT: // kill: def $q0 killed $h0
|
|
; GISELO0-NEXT: mov w8, v0.s[0]
|
|
; GISELO0-NEXT: // kill: def $x8 killed $w8
|
|
; GISELO0-NEXT: fmov d0, x8
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %0)
|
|
ret <1 x i64> %1
|
|
}
|
|
|
|
declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>)
|
|
|
|
define <4 x i32> @popcount4x32(<4 x i32> %0) {
|
|
; CHECKO0-LABEL: popcount4x32:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.4s, v0.8h
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount4x32:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-NEXT: uaddlp v0.4s, v0.8h
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount4x32:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: movi v1.16b, #1
|
|
; DOT-NEXT: cnt v2.16b, v0.16b
|
|
; DOT-NEXT: movi v0.2d, #0000000000000000
|
|
; DOT-NEXT: udot v0.4s, v1.16b, v2.16b
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount4x32:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: ptrue p0.s, vl4
|
|
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
|
|
; SVE-NEXT: cnt z0.s, p0/m, z0.s
|
|
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount4x32:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: rev64 v0.16b, v0.16b
|
|
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
|
|
; BE-NEXT: cnt v0.16b, v0.16b
|
|
; BE-NEXT: uaddlp v0.8h, v0.16b
|
|
; BE-NEXT: uaddlp v0.4s, v0.8h
|
|
; BE-NEXT: rev64 v0.4s, v0.4s
|
|
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount4x32:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.4s, v0.8h
|
|
; GISELO0-NEXT: ret
|
|
;
|
|
; NEON-GISEL-LABEL: popcount4x32:
|
|
; NEON-GISEL: // %bb.0: // %Entry
|
|
; NEON-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-GISEL-NEXT: uaddlp v0.4s, v0.8h
|
|
; NEON-GISEL-NEXT: ret
|
|
;
|
|
; DOT-GISEL-LABEL: popcount4x32:
|
|
; DOT-GISEL: // %bb.0: // %Entry
|
|
; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
|
|
; DOT-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-GISEL-NEXT: movi v2.16b, #1
|
|
; DOT-GISEL-NEXT: udot v1.4s, v2.16b, v0.16b
|
|
; DOT-GISEL-NEXT: mov v0.16b, v1.16b
|
|
; DOT-GISEL-NEXT: ret
|
|
;
|
|
; SVE-GISEL-LABEL: popcount4x32:
|
|
; SVE-GISEL: // %bb.0: // %Entry
|
|
; SVE-GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; SVE-GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; SVE-GISEL-NEXT: uaddlp v0.4s, v0.8h
|
|
; SVE-GISEL-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %0)
|
|
ret <4 x i32> %1
|
|
}
|
|
|
|
declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>)
|
|
|
|
define <2 x i32> @popcount2x32(<2 x i32> %0) {
|
|
; CHECKO0-LABEL: popcount2x32:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.8b, v0.8b
|
|
; CHECKO0-NEXT: uaddlp v0.4h, v0.8b
|
|
; CHECKO0-NEXT: uaddlp v0.2s, v0.4h
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount2x32:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.8b, v0.8b
|
|
; NEON-NEXT: uaddlp v0.4h, v0.8b
|
|
; NEON-NEXT: uaddlp v0.2s, v0.4h
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount2x32:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: movi v1.2d, #0000000000000000
|
|
; DOT-NEXT: cnt v0.8b, v0.8b
|
|
; DOT-NEXT: movi v2.8b, #1
|
|
; DOT-NEXT: udot v1.2s, v2.8b, v0.8b
|
|
; DOT-NEXT: fmov d0, d1
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount2x32:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: ptrue p0.s, vl2
|
|
; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
|
|
; SVE-NEXT: cnt z0.s, p0/m, z0.s
|
|
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount2x32:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: rev64 v0.8b, v0.8b
|
|
; BE-NEXT: cnt v0.8b, v0.8b
|
|
; BE-NEXT: uaddlp v0.4h, v0.8b
|
|
; BE-NEXT: uaddlp v0.2s, v0.4h
|
|
; BE-NEXT: rev64 v0.2s, v0.2s
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount2x32:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.8b, v0.8b
|
|
; GISELO0-NEXT: uaddlp v0.4h, v0.8b
|
|
; GISELO0-NEXT: uaddlp v0.2s, v0.4h
|
|
; GISELO0-NEXT: ret
|
|
;
|
|
; NEON-GISEL-LABEL: popcount2x32:
|
|
; NEON-GISEL: // %bb.0: // %Entry
|
|
; NEON-GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; NEON-GISEL-NEXT: uaddlp v0.4h, v0.8b
|
|
; NEON-GISEL-NEXT: uaddlp v0.2s, v0.4h
|
|
; NEON-GISEL-NEXT: ret
|
|
;
|
|
; DOT-GISEL-LABEL: popcount2x32:
|
|
; DOT-GISEL: // %bb.0: // %Entry
|
|
; DOT-GISEL-NEXT: movi v1.2d, #0000000000000000
|
|
; DOT-GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; DOT-GISEL-NEXT: movi v2.8b, #1
|
|
; DOT-GISEL-NEXT: udot v1.2s, v2.8b, v0.8b
|
|
; DOT-GISEL-NEXT: fmov d0, d1
|
|
; DOT-GISEL-NEXT: ret
|
|
;
|
|
; SVE-GISEL-LABEL: popcount2x32:
|
|
; SVE-GISEL: // %bb.0: // %Entry
|
|
; SVE-GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; SVE-GISEL-NEXT: uaddlp v0.4h, v0.8b
|
|
; SVE-GISEL-NEXT: uaddlp v0.2s, v0.4h
|
|
; SVE-GISEL-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %0)
|
|
ret <2 x i32> %1
|
|
}
|
|
|
|
declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>)
|
|
|
|
define <8 x i16> @popcount8x16(<8 x i16> %0) {
|
|
; CHECKO0-LABEL: popcount8x16:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.16b, v0.16b
|
|
; CHECKO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount8x16:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.16b, v0.16b
|
|
; NEON-NEXT: uaddlp v0.8h, v0.16b
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount8x16:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: cnt v0.16b, v0.16b
|
|
; DOT-NEXT: uaddlp v0.8h, v0.16b
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount8x16:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: ptrue p0.h, vl8
|
|
; SVE-NEXT: // kill: def $q0 killed $q0 def $z0
|
|
; SVE-NEXT: cnt z0.h, p0/m, z0.h
|
|
; SVE-NEXT: // kill: def $q0 killed $q0 killed $z0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount8x16:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: rev64 v0.16b, v0.16b
|
|
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
|
|
; BE-NEXT: cnt v0.16b, v0.16b
|
|
; BE-NEXT: uaddlp v0.8h, v0.16b
|
|
; BE-NEXT: rev64 v0.8h, v0.8h
|
|
; BE-NEXT: ext v0.16b, v0.16b, v0.16b, #8
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount8x16:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: cnt v0.16b, v0.16b
|
|
; GISEL-NEXT: uaddlp v0.8h, v0.16b
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount8x16:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.16b, v0.16b
|
|
; GISELO0-NEXT: uaddlp v0.8h, v0.16b
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %0)
|
|
ret <8 x i16> %1
|
|
}
|
|
|
|
declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>)
|
|
|
|
define <4 x i16> @popcount4x16(<4 x i16> %0) {
|
|
; CHECKO0-LABEL: popcount4x16:
|
|
; CHECKO0: // %bb.0: // %Entry
|
|
; CHECKO0-NEXT: cnt v0.8b, v0.8b
|
|
; CHECKO0-NEXT: uaddlp v0.4h, v0.8b
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: popcount4x16:
|
|
; NEON: // %bb.0: // %Entry
|
|
; NEON-NEXT: cnt v0.8b, v0.8b
|
|
; NEON-NEXT: uaddlp v0.4h, v0.8b
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: popcount4x16:
|
|
; DOT: // %bb.0: // %Entry
|
|
; DOT-NEXT: cnt v0.8b, v0.8b
|
|
; DOT-NEXT: uaddlp v0.4h, v0.8b
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: popcount4x16:
|
|
; SVE: // %bb.0: // %Entry
|
|
; SVE-NEXT: ptrue p0.h, vl4
|
|
; SVE-NEXT: // kill: def $d0 killed $d0 def $z0
|
|
; SVE-NEXT: cnt z0.h, p0/m, z0.h
|
|
; SVE-NEXT: // kill: def $d0 killed $d0 killed $z0
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: popcount4x16:
|
|
; BE: // %bb.0: // %Entry
|
|
; BE-NEXT: rev64 v0.8b, v0.8b
|
|
; BE-NEXT: cnt v0.8b, v0.8b
|
|
; BE-NEXT: uaddlp v0.4h, v0.8b
|
|
; BE-NEXT: rev64 v0.4h, v0.4h
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: popcount4x16:
|
|
; GISEL: // %bb.0: // %Entry
|
|
; GISEL-NEXT: cnt v0.8b, v0.8b
|
|
; GISEL-NEXT: uaddlp v0.4h, v0.8b
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: popcount4x16:
|
|
; GISELO0: // %bb.0: // %Entry
|
|
; GISELO0-NEXT: cnt v0.8b, v0.8b
|
|
; GISELO0-NEXT: uaddlp v0.4h, v0.8b
|
|
; GISELO0-NEXT: ret
|
|
Entry:
|
|
%1 = tail call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %0)
|
|
ret <4 x i16> %1
|
|
}
|
|
|
|
define i32 @ctpop_into_extract(ptr %p) {
|
|
; CHECKO0-LABEL: ctpop_into_extract:
|
|
; CHECKO0: // %bb.0:
|
|
; CHECKO0-NEXT: mov w8, #-1 // =0xffffffff
|
|
; CHECKO0-NEXT: // implicit-def: $d2
|
|
; CHECKO0-NEXT: fmov s2, w8
|
|
; CHECKO0-NEXT: ldr d0, [x0]
|
|
; CHECKO0-NEXT: fmov s1, s0
|
|
; CHECKO0-NEXT: fmov w8, s1
|
|
; CHECKO0-NEXT: fmov s1, w8
|
|
; CHECKO0-NEXT: // kill: def $d1 killed $s1
|
|
; CHECKO0-NEXT: cnt v1.8b, v1.8b
|
|
; CHECKO0-NEXT: uaddlv h1, v1.8b
|
|
; CHECKO0-NEXT: // kill: def $q1 killed $h1
|
|
; CHECKO0-NEXT: // kill: def $s1 killed $s1 killed $q1
|
|
; CHECKO0-NEXT: fmov w8, s1
|
|
; CHECKO0-NEXT: // implicit-def: $q1
|
|
; CHECKO0-NEXT: fmov d1, d2
|
|
; CHECKO0-NEXT: mov v1.s[1], w8
|
|
; CHECKO0-NEXT: // kill: def $d1 killed $d1 killed $q1
|
|
; CHECKO0-NEXT: sub v0.2s, v0.2s, v1.2s
|
|
; CHECKO0-NEXT: str d0, [x0]
|
|
; CHECKO0-NEXT: mov w0, wzr
|
|
; CHECKO0-NEXT: ret
|
|
;
|
|
; NEON-LABEL: ctpop_into_extract:
|
|
; NEON: // %bb.0:
|
|
; NEON-NEXT: ldr d0, [x0]
|
|
; NEON-NEXT: movi v2.2d, #0xffffffffffffffff
|
|
; NEON-NEXT: mov x8, x0
|
|
; NEON-NEXT: mov w0, wzr
|
|
; NEON-NEXT: fmov w9, s0
|
|
; NEON-NEXT: fmov s1, w9
|
|
; NEON-NEXT: cnt v1.8b, v1.8b
|
|
; NEON-NEXT: addv b1, v1.8b
|
|
; NEON-NEXT: mov v2.s[1], v1.s[0]
|
|
; NEON-NEXT: sub v0.2s, v0.2s, v2.2s
|
|
; NEON-NEXT: str d0, [x8]
|
|
; NEON-NEXT: ret
|
|
;
|
|
; DOT-LABEL: ctpop_into_extract:
|
|
; DOT: // %bb.0:
|
|
; DOT-NEXT: ldr d0, [x0]
|
|
; DOT-NEXT: movi v2.2d, #0xffffffffffffffff
|
|
; DOT-NEXT: mov x8, x0
|
|
; DOT-NEXT: mov w0, wzr
|
|
; DOT-NEXT: fmov w9, s0
|
|
; DOT-NEXT: fmov s1, w9
|
|
; DOT-NEXT: cnt v1.8b, v1.8b
|
|
; DOT-NEXT: addv b1, v1.8b
|
|
; DOT-NEXT: mov v2.s[1], v1.s[0]
|
|
; DOT-NEXT: sub v0.2s, v0.2s, v2.2s
|
|
; DOT-NEXT: str d0, [x8]
|
|
; DOT-NEXT: ret
|
|
;
|
|
; SVE-LABEL: ctpop_into_extract:
|
|
; SVE: // %bb.0:
|
|
; SVE-NEXT: ldr d0, [x0]
|
|
; SVE-NEXT: ptrue p0.s
|
|
; SVE-NEXT: movi v2.2d, #0xffffffffffffffff
|
|
; SVE-NEXT: mov x8, x0
|
|
; SVE-NEXT: mov w0, wzr
|
|
; SVE-NEXT: movprfx z1, z0
|
|
; SVE-NEXT: cnt z1.s, p0/m, z0.s
|
|
; SVE-NEXT: mov v2.s[1], v1.s[0]
|
|
; SVE-NEXT: sub v0.2s, v0.2s, v2.2s
|
|
; SVE-NEXT: str d0, [x8]
|
|
; SVE-NEXT: ret
|
|
;
|
|
; BE-LABEL: ctpop_into_extract:
|
|
; BE: // %bb.0:
|
|
; BE-NEXT: ld1 { v0.2s }, [x0]
|
|
; BE-NEXT: movi v2.2d, #0xffffffffffffffff
|
|
; BE-NEXT: mov x8, x0
|
|
; BE-NEXT: mov w0, wzr
|
|
; BE-NEXT: fmov w9, s0
|
|
; BE-NEXT: fmov s1, w9
|
|
; BE-NEXT: cnt v1.8b, v1.8b
|
|
; BE-NEXT: addv b1, v1.8b
|
|
; BE-NEXT: mov v2.s[1], v1.s[0]
|
|
; BE-NEXT: sub v0.2s, v0.2s, v2.2s
|
|
; BE-NEXT: st1 { v0.2s }, [x8]
|
|
; BE-NEXT: ret
|
|
;
|
|
; GISEL-LABEL: ctpop_into_extract:
|
|
; GISEL: // %bb.0:
|
|
; GISEL-NEXT: ldr d0, [x0]
|
|
; GISEL-NEXT: mov x8, x0
|
|
; GISEL-NEXT: mov w0, wzr
|
|
; GISEL-NEXT: fmov w9, s0
|
|
; GISEL-NEXT: fmov s1, w9
|
|
; GISEL-NEXT: mov w9, #-1 // =0xffffffff
|
|
; GISEL-NEXT: fmov s2, w9
|
|
; GISEL-NEXT: cnt v1.8b, v1.8b
|
|
; GISEL-NEXT: uaddlv h1, v1.8b
|
|
; GISEL-NEXT: mov v2.s[1], v1.s[0]
|
|
; GISEL-NEXT: sub v0.2s, v0.2s, v2.2s
|
|
; GISEL-NEXT: str d0, [x8]
|
|
; GISEL-NEXT: ret
|
|
;
|
|
; GISELO0-LABEL: ctpop_into_extract:
|
|
; GISELO0: // %bb.0:
|
|
; GISELO0-NEXT: mov w8, #-1 // =0xffffffff
|
|
; GISELO0-NEXT: // implicit-def: $d2
|
|
; GISELO0-NEXT: fmov s2, w8
|
|
; GISELO0-NEXT: ldr d0, [x0]
|
|
; GISELO0-NEXT: fmov s1, s0
|
|
; GISELO0-NEXT: fmov w8, s1
|
|
; GISELO0-NEXT: fmov s1, w8
|
|
; GISELO0-NEXT: // kill: def $d1 killed $s1
|
|
; GISELO0-NEXT: cnt v1.8b, v1.8b
|
|
; GISELO0-NEXT: uaddlv h1, v1.8b
|
|
; GISELO0-NEXT: // kill: def $q1 killed $h1
|
|
; GISELO0-NEXT: // kill: def $s1 killed $s1 killed $q1
|
|
; GISELO0-NEXT: fmov w8, s1
|
|
; GISELO0-NEXT: // implicit-def: $q1
|
|
; GISELO0-NEXT: fmov d1, d2
|
|
; GISELO0-NEXT: mov v1.s[1], w8
|
|
; GISELO0-NEXT: // kill: def $d1 killed $d1 killed $q1
|
|
; GISELO0-NEXT: sub v0.2s, v0.2s, v1.2s
|
|
; GISELO0-NEXT: str d0, [x0]
|
|
; GISELO0-NEXT: mov w0, wzr
|
|
; GISELO0-NEXT: ret
|
|
%1 = load <2 x i32>, ptr %p, align 4
|
|
%2 = extractelement <2 x i32> %1, i64 0
|
|
%3 = call i32 @llvm.ctpop.i32(i32 %2)
|
|
%4 = insertelement <2 x i32> <i32 -1, i32 poison>, i32 %3, i64 1
|
|
%5 = sub <2 x i32> %1, %4
|
|
store <2 x i32> %5, ptr %p, align 4
|
|
ret i32 0
|
|
}
|
|
|
|
declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>)
|
|
;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
|
|
; CHECK: {{.*}}
|