From 3fac23505ffdb3b7ff7a61fc8beca2aabc2cb6c8 Mon Sep 17 00:00:00 2001
From: David Green <david.green@arm.com>
Date: Sat, 15 Mar 2025 22:23:44 +0000
Subject: [PATCH] [AArch64][GlobalISel] Add cttz, ctlz and ctpop test coverage.
 NFC

---
 llvm/test/CodeGen/AArch64/arm64-vclz.ll    | 254 ------
 llvm/test/CodeGen/AArch64/arm64-vpopcnt.ll | 104 ---
 llvm/test/CodeGen/AArch64/ctlz.ll          | 872 +++++++++++++++++++++
 llvm/test/CodeGen/AArch64/ctpop.ll         | 377 +++++++++
 llvm/test/CodeGen/AArch64/cttz.ll          | 530 +++++++++++++
 llvm/test/CodeGen/AArch64/vec_cttz.ll      | 124 ---
 6 files changed, 1779 insertions(+), 482 deletions(-)
 delete mode 100644 llvm/test/CodeGen/AArch64/arm64-vclz.ll
 delete mode 100644 llvm/test/CodeGen/AArch64/arm64-vpopcnt.ll
 create mode 100644 llvm/test/CodeGen/AArch64/ctlz.ll
 create mode 100644 llvm/test/CodeGen/AArch64/ctpop.ll
 create mode 100644 llvm/test/CodeGen/AArch64/cttz.ll
 delete mode 100644 llvm/test/CodeGen/AArch64/vec_cttz.ll

diff --git a/llvm/test/CodeGen/AArch64/arm64-vclz.ll b/llvm/test/CodeGen/AArch64/arm64-vclz.ll
deleted file mode 100644
index c65e75c89e8d..000000000000
--- a/llvm/test/CodeGen/AArch64/arm64-vclz.ll
+++ /dev/null
@@ -1,254 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple | FileCheck %s --check-prefixes=CHECK,CHECK-SD
-; RUN: llc < %s -mtriple=arm64-eabi -aarch64-neon-syntax=apple -global-isel | FileCheck %s --check-prefixes=CHECK,CHECK-GI
-
-define <8 x i8> @test_vclz_u8(<8 x i8> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclz_u8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.8b v0, v0
-; CHECK-NEXT:    ret
-  %vclz.i = tail call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %a, i1 false) nounwind
-  ret <8 x i8> %vclz.i
-}
-
-define <8 x i8> @test_vclz_s8(<8 x i8> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclz_s8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.8b v0, v0
-; CHECK-NEXT:    ret
-  %vclz.i = tail call <8 x i8> @llvm.ctlz.v8i8(<8 x i8> %a, i1 false) nounwind
-  ret <8 x i8> %vclz.i
-}
-
-define <4 x i16> @test_vclz_u16(<4 x i16> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclz_u16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.4h v0, v0
-; CHECK-NEXT:    ret
-  %vclz1.i = tail call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %a, i1 false) nounwind
-  ret <4 x i16> %vclz1.i
-}
-
-define <4 x i16> @test_vclz_s16(<4 x i16> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclz_s16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.4h v0, v0
-; CHECK-NEXT:    ret
-  %vclz1.i = tail call <4 x i16> @llvm.ctlz.v4i16(<4 x i16> %a, i1 false) nounwind
-  ret <4 x i16> %vclz1.i
-}
-
-define <2 x i32> @test_vclz_u32(<2 x i32> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclz_u32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.2s v0, v0
-; CHECK-NEXT:    ret
-  %vclz1.i = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false) nounwind
-  ret <2 x i32> %vclz1.i
-}
-
-define <2 x i32> @test_vclz_s32(<2 x i32> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclz_s32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.2s v0, v0
-; CHECK-NEXT:    ret
-  %vclz1.i = tail call <2 x i32> @llvm.ctlz.v2i32(<2 x i32> %a, i1 false) nounwind
-  ret <2 x i32> %vclz1.i
-}
-
-define <1 x i64> @test_vclz_u64(<1 x i64> %a) nounwind readnone ssp {
-; CHECK-SD-LABEL: test_vclz_u64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ushr d1, d0, #1
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #2
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #4
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #8
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #16
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #32
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    mvn.8b v0, v0
-; CHECK-SD-NEXT:    cnt.8b v0, v0
-; CHECK-SD-NEXT:    uaddlp.4h v0, v0
-; CHECK-SD-NEXT:    uaddlp.2s v0, v0
-; CHECK-SD-NEXT:    uaddlp.1d v0, v0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vclz_u64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    clz x8, x8
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    ret
-  %vclz1.i = tail call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %a, i1 false) nounwind
-  ret <1 x i64> %vclz1.i
-}
-
-define <1 x i64> @test_vclz_s64(<1 x i64> %a) nounwind readnone ssp {
-; CHECK-SD-LABEL: test_vclz_s64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ushr d1, d0, #1
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #2
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #4
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #8
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #16
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    ushr d1, d0, #32
-; CHECK-SD-NEXT:    orr.8b v0, v0, v1
-; CHECK-SD-NEXT:    mvn.8b v0, v0
-; CHECK-SD-NEXT:    cnt.8b v0, v0
-; CHECK-SD-NEXT:    uaddlp.4h v0, v0
-; CHECK-SD-NEXT:    uaddlp.2s v0, v0
-; CHECK-SD-NEXT:    uaddlp.1d v0, v0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vclz_s64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    clz x8, x8
-; CHECK-GI-NEXT:    fmov d0, x8
-; CHECK-GI-NEXT:    ret
-  %vclz1.i = tail call <1 x i64> @llvm.ctlz.v1i64(<1 x i64> %a, i1 false) nounwind
-  ret <1 x i64> %vclz1.i
-}
-
-define <16 x i8> @test_vclzq_u8(<16 x i8> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclzq_u8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.16b v0, v0
-; CHECK-NEXT:    ret
-  %vclz.i = tail call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false) nounwind
-  ret <16 x i8> %vclz.i
-}
-
-define <16 x i8> @test_vclzq_s8(<16 x i8> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclzq_s8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.16b v0, v0
-; CHECK-NEXT:    ret
-  %vclz.i = tail call <16 x i8> @llvm.ctlz.v16i8(<16 x i8> %a, i1 false) nounwind
-  ret <16 x i8> %vclz.i
-}
-
-define <8 x i16> @test_vclzq_u16(<8 x i16> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclzq_u16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.8h v0, v0
-; CHECK-NEXT:    ret
-  %vclz1.i = tail call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false) nounwind
-  ret <8 x i16> %vclz1.i
-}
-
-define <8 x i16> @test_vclzq_s16(<8 x i16> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclzq_s16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.8h v0, v0
-; CHECK-NEXT:    ret
-  %vclz1.i = tail call <8 x i16> @llvm.ctlz.v8i16(<8 x i16> %a, i1 false) nounwind
-  ret <8 x i16> %vclz1.i
-}
-
-define <4 x i32> @test_vclzq_u32(<4 x i32> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclzq_u32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.4s v0, v0
-; CHECK-NEXT:    ret
-  %vclz1.i = tail call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false) nounwind
-  ret <4 x i32> %vclz1.i
-}
-
-define <4 x i32> @test_vclzq_s32(<4 x i32> %a) nounwind readnone ssp {
-; CHECK-LABEL: test_vclzq_s32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    clz.4s v0, v0
-; CHECK-NEXT:    ret
-  %vclz1.i = tail call <4 x i32> @llvm.ctlz.v4i32(<4 x i32> %a, i1 false) nounwind
-  ret <4 x i32> %vclz1.i
-}
-
-define <2 x i64> @test_vclzq_u64(<2 x i64> %a) nounwind readnone ssp {
-; CHECK-SD-LABEL: test_vclzq_u64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #1
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #2
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #4
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #8
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #16
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #32
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    mvn.16b v0, v0
-; CHECK-SD-NEXT:    cnt.16b v0, v0
-; CHECK-SD-NEXT:    uaddlp.8h v0, v0
-; CHECK-SD-NEXT:    uaddlp.4s v0, v0
-; CHECK-SD-NEXT:    uaddlp.2d v0, v0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vclzq_u64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    mov.d x9, v0[1]
-; CHECK-GI-NEXT:    clz x8, x8
-; CHECK-GI-NEXT:    mov.d v0[0], x8
-; CHECK-GI-NEXT:    clz x8, x9
-; CHECK-GI-NEXT:    mov.d v0[1], x8
-; CHECK-GI-NEXT:    ret
-  %vclz1.i = tail call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 false) nounwind
-  ret <2 x i64> %vclz1.i
-}
-
-define <2 x i64> @test_vclzq_s64(<2 x i64> %a) nounwind readnone ssp {
-; CHECK-SD-LABEL: test_vclzq_s64:
-; CHECK-SD:       // %bb.0:
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #1
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #2
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #4
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #8
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #16
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    ushr.2d v1, v0, #32
-; CHECK-SD-NEXT:    orr.16b v0, v0, v1
-; CHECK-SD-NEXT:    mvn.16b v0, v0
-; CHECK-SD-NEXT:    cnt.16b v0, v0
-; CHECK-SD-NEXT:    uaddlp.8h v0, v0
-; CHECK-SD-NEXT:    uaddlp.4s v0, v0
-; CHECK-SD-NEXT:    uaddlp.2d v0, v0
-; CHECK-SD-NEXT:    ret
-;
-; CHECK-GI-LABEL: test_vclzq_s64:
-; CHECK-GI:       // %bb.0:
-; CHECK-GI-NEXT:    fmov x8, d0
-; CHECK-GI-NEXT:    mov.d x9, v0[1]
-; CHECK-GI-NEXT:    clz x8, x8
-; CHECK-GI-NEXT:    mov.d v0[0], x8
-; CHECK-GI-NEXT:    clz x8, x9
-; CHECK-GI-NEXT:    mov.d v0[1], x8
-; CHECK-GI-NEXT:    ret
-  %vclz1.i = tail call <2 x i64> @llvm.ctlz.v2i64(<2 x i64> %a, i1 false) nounwind
-  ret <2 x i64> %vclz1.i
-}
-
-declare <2 x i64> @llvm.ctlz.v2i64(<2 x i64>, i1) nounwind readnone
-declare <4 x i32> @llvm.ctlz.v4i32(<4 x i32>, i1) nounwind readnone
-declare <8 x i16> @llvm.ctlz.v8i16(<8 x i16>, i1) nounwind readnone
-declare <16 x i8> @llvm.ctlz.v16i8(<16 x i8>, i1) nounwind readnone
-declare <1 x i64> @llvm.ctlz.v1i64(<1 x i64>, i1) nounwind readnone
-declare <2 x i32> @llvm.ctlz.v2i32(<2 x i32>, i1) nounwind readnone
-declare <4 x i16> @llvm.ctlz.v4i16(<4 x i16>, i1) nounwind readnone
-declare <8 x i8> @llvm.ctlz.v8i8(<8 x i8>, i1) nounwind readnone
diff --git a/llvm/test/CodeGen/AArch64/arm64-vpopcnt.ll b/llvm/test/CodeGen/AArch64/arm64-vpopcnt.ll
deleted file mode 100644
index 6fe1176eaa87..000000000000
--- a/llvm/test/CodeGen/AArch64/arm64-vpopcnt.ll
+++ /dev/null
@@ -1,104 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64-apple- -mcpu=cyclone | FileCheck %s
-
-; The non-byte ones used to fail with "Cannot select"
-
-define <8 x i8> @ctpopv8i8(<8 x i8> %x) nounwind readnone {
-; CHECK-LABEL: ctpopv8i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    ret
-  %cnt = tail call <8 x i8> @llvm.ctpop.v8i8(<8 x i8> %x)
-  ret <8 x i8> %cnt
-}
-
-declare <8 x i8> @llvm.ctpop.v8i8(<8 x i8>) nounwind readnone
-
-define <4 x i16> @ctpopv4i16(<4 x i16> %x) nounwind readnone {
-; CHECK-LABEL: ctpopv4i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    ret
-  %cnt = tail call <4 x i16> @llvm.ctpop.v4i16(<4 x i16> %x)
-  ret <4 x i16> %cnt
-}
-
-declare <4 x i16> @llvm.ctpop.v4i16(<4 x i16>) nounwind readnone
-
-define <2 x i32> @ctpopv2i32(<2 x i32> %x) nounwind readnone {
-; CHECK-LABEL: ctpopv2i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    ret
-  %cnt = tail call <2 x i32> @llvm.ctpop.v2i32(<2 x i32> %x)
-  ret <2 x i32> %cnt
-}
-
-declare <2 x i32> @llvm.ctpop.v2i32(<2 x i32>) nounwind readnone
-
-define <1 x i64> @ctpopv1i64(<1 x i64> %x) nounwind readnone {
-; CHECK-LABEL: ctpopv1i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    uaddlp v0.1d, v0.2s
-; CHECK-NEXT:    ret
-  %cnt = tail call <1 x i64> @llvm.ctpop.v1i64(<1 x i64> %x)
-  ret <1 x i64> %cnt
-}
-
-declare <1 x i64> @llvm.ctpop.v1i64(<1 x i64>) nounwind readnone
-
-define <16 x i8> @ctpopv16i8(<16 x i8> %x) nounwind readnone {
-; CHECK-LABEL: ctpopv16i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    ret
-  %cnt = tail call <16 x i8> @llvm.ctpop.v16i8(<16 x i8> %x)
-  ret <16 x i8> %cnt
-}
-
-declare <16 x i8> @llvm.ctpop.v16i8(<16 x i8>) nounwind readnone
-
-define <8 x i16> @ctpopv8i16(<8 x i16> %x) nounwind readnone {
-; CHECK-LABEL: ctpopv8i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    ret
-  %cnt = tail call <8 x i16> @llvm.ctpop.v8i16(<8 x i16> %x)
-  ret <8 x i16> %cnt
-}
-
-declare <8 x i16> @llvm.ctpop.v8i16(<8 x i16>) nounwind readnone
-
-define <4 x i32> @ctpopv4i32(<4 x i32> %x) nounwind readnone {
-; CHECK-LABEL: ctpopv4i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    ret
-  %cnt = tail call <4 x i32> @llvm.ctpop.v4i32(<4 x i32> %x)
-  ret <4 x i32> %cnt
-}
-
-declare <4 x i32> @llvm.ctpop.v4i32(<4 x i32>) nounwind readnone
-
-define <2 x i64> @ctpopv2i64(<2 x i64> %x) nounwind readnone {
-; CHECK-LABEL: ctpopv2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
-; CHECK-NEXT:    ret
-  %cnt = tail call <2 x i64> @llvm.ctpop.v2i64(<2 x i64> %x)
-  ret <2 x i64> %cnt
-}
-
-declare <2 x i64> @llvm.ctpop.v2i64(<2 x i64>) nounwind readnone
diff --git a/llvm/test/CodeGen/AArch64/ctlz.ll b/llvm/test/CodeGen/AArch64/ctlz.ll
new file mode 100644
index 000000000000..437e3d5ff75c
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ctlz.ll
@@ -0,0 +1,872 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+
+define void @v2i8(ptr %p1) {
+; CHECK-SD-LABEL: v2i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrb w8, [x0]
+; CHECK-SD-NEXT:    ldrb w9, [x0, #1]
+; CHECK-SD-NEXT:    movi v0.2s, #24
+; CHECK-SD-NEXT:    fmov s1, w8
+; CHECK-SD-NEXT:    mov v1.s[1], w9
+; CHECK-SD-NEXT:    clz v1.2s, v1.2s
+; CHECK-SD-NEXT:    sub v0.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    strb w9, [x0]
+; CHECK-SD-NEXT:    strb w8, [x0, #1]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    ldrb w9, [x0, #1]
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    strb w8, [x0]
+; CHECK-GI-NEXT:    strb w9, [x0, #1]
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = load <2 x i8>, ptr %p1
+  %s = call <2 x i8> @llvm.ctlz(<2 x i8> %d, i1 false)
+  store <2 x i8> %s, ptr %p1
+  ret void
+}
+
+define void @v3i8(ptr %p1) {
+; CHECK-SD-LABEL: v3i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    sub sp, sp, #16
+; CHECK-SD-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-SD-NEXT:    movi v0.4h, #8
+; CHECK-SD-NEXT:    ldr s1, [x0]
+; CHECK-SD-NEXT:    zip1 v1.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    bic v1.4h, #255, lsl #8
+; CHECK-SD-NEXT:    clz v1.4h, v1.4h
+; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-SD-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    umov w8, v0.h[2]
+; CHECK-SD-NEXT:    str s1, [sp, #12]
+; CHECK-SD-NEXT:    ldrh w9, [sp, #12]
+; CHECK-SD-NEXT:    strb w8, [x0, #2]
+; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    add sp, sp, #16
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrb w8, [x0]
+; CHECK-GI-NEXT:    ldrb w9, [x0, #1]
+; CHECK-GI-NEXT:    ldrb w10, [x0, #2]
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w10
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    strb w8, [x0]
+; CHECK-GI-NEXT:    sub w8, w10, #24
+; CHECK-GI-NEXT:    strb w9, [x0, #1]
+; CHECK-GI-NEXT:    strb w8, [x0, #2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = load <3 x i8>, ptr %p1
+  %s = call <3 x i8> @llvm.ctlz(<3 x i8> %d, i1 false)
+  store <3 x i8> %s, ptr %p1
+  ret void
+}
+
+define void @v4i8(ptr %p1) {
+; CHECK-SD-LABEL: v4i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr s1, [x0]
+; CHECK-SD-NEXT:    movi v0.4h, #8
+; CHECK-SD-NEXT:    ushll v1.8h, v1.8b, #0
+; CHECK-SD-NEXT:    clz v1.4h, v1.4h
+; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-SD-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldr w8, [x0]
+; CHECK-GI-NEXT:    fmov s0, w8
+; CHECK-GI-NEXT:    uxtb w8, w8
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    mov b1, v0.b[1]
+; CHECK-GI-NEXT:    mov b2, v0.b[2]
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    mov b0, v0.b[3]
+; CHECK-GI-NEXT:    fmov w9, s1
+; CHECK-GI-NEXT:    fmov w10, s2
+; CHECK-GI-NEXT:    fmov s1, w8
+; CHECK-GI-NEXT:    uxtb w9, w9
+; CHECK-GI-NEXT:    uxtb w8, w10
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    mov v1.b[1], w9
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    uxtb w9, w9
+; CHECK-GI-NEXT:    mov v1.b[2], w8
+; CHECK-GI-NEXT:    clz w8, w9
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    mov v1.b[3], w8
+; CHECK-GI-NEXT:    fmov w8, s1
+; CHECK-GI-NEXT:    str w8, [x0]
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = load <4 x i8>, ptr %p1
+  %s = call <4 x i8> @llvm.ctlz(<4 x i8> %d, i1 false)
+  store <4 x i8> %s, ptr %p1
+  ret void
+}
+
+define <8 x i8> @v8i8(<8 x i8> %d) {
+; CHECK-LABEL: v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    clz v0.8b, v0.8b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <8 x i8> @llvm.ctlz(<8 x i8> %d, i1 false)
+  ret <8 x i8> %s
+}
+
+define <16 x i8> @v16i8(<16 x i8> %d) {
+; CHECK-LABEL: v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    clz v0.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <16 x i8> @llvm.ctlz(<16 x i8> %d, i1 false)
+  ret <16 x i8> %s
+}
+
+define <32 x i8> @v32i8(<32 x i8> %d) {
+; CHECK-SD-LABEL: v32i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    clz v0.16b, v0.16b
+; CHECK-SD-NEXT:    clz v1.16b, v1.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v32i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    umov w9, v0.b[0]
+; CHECK-GI-NEXT:    umov w11, v1.b[0]
+; CHECK-GI-NEXT:    umov w10, v0.b[1]
+; CHECK-GI-NEXT:    umov w13, v1.b[1]
+; CHECK-GI-NEXT:    umov w8, v0.b[2]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w11, w11
+; CHECK-GI-NEXT:    clz w10, w10
+; CHECK-GI-NEXT:    sub w14, w9, #24
+; CHECK-GI-NEXT:    sub w12, w11, #24
+; CHECK-GI-NEXT:    clz w11, w13
+; CHECK-GI-NEXT:    fmov s2, w14
+; CHECK-GI-NEXT:    fmov s3, w12
+; CHECK-GI-NEXT:    umov w9, v1.b[2]
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    sub w11, w11, #24
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    mov v2.b[1], w10
+; CHECK-GI-NEXT:    mov v3.b[1], w11
+; CHECK-GI-NEXT:    umov w10, v0.b[3]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    umov w11, v1.b[3]
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    mov v2.b[2], w8
+; CHECK-GI-NEXT:    mov v3.b[2], w9
+; CHECK-GI-NEXT:    clz w8, w10
+; CHECK-GI-NEXT:    umov w9, v0.b[4]
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[4]
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[3], w8
+; CHECK-GI-NEXT:    mov v3.b[3], w10
+; CHECK-GI-NEXT:    umov w8, v0.b[5]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[5]
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[4], w9
+; CHECK-GI-NEXT:    mov v3.b[4], w10
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    umov w9, v0.b[6]
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[6]
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[5], w8
+; CHECK-GI-NEXT:    mov v3.b[5], w10
+; CHECK-GI-NEXT:    umov w8, v0.b[7]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[7]
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[6], w9
+; CHECK-GI-NEXT:    mov v3.b[6], w10
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    umov w9, v0.b[8]
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[8]
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[7], w8
+; CHECK-GI-NEXT:    mov v3.b[7], w10
+; CHECK-GI-NEXT:    umov w8, v0.b[9]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[9]
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[8], w9
+; CHECK-GI-NEXT:    mov v3.b[8], w10
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    umov w9, v0.b[10]
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[10]
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[9], w8
+; CHECK-GI-NEXT:    mov v3.b[9], w10
+; CHECK-GI-NEXT:    umov w8, v0.b[11]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[11]
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[10], w9
+; CHECK-GI-NEXT:    mov v3.b[10], w10
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    umov w9, v0.b[12]
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[12]
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[11], w8
+; CHECK-GI-NEXT:    mov v3.b[11], w10
+; CHECK-GI-NEXT:    umov w8, v0.b[13]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[13]
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[12], w9
+; CHECK-GI-NEXT:    mov v3.b[12], w10
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    umov w9, v0.b[14]
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.b[14]
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w10, w10, #24
+; CHECK-GI-NEXT:    mov v2.b[13], w8
+; CHECK-GI-NEXT:    mov v3.b[13], w10
+; CHECK-GI-NEXT:    umov w8, v0.b[15]
+; CHECK-GI-NEXT:    umov w10, v1.b[15]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w11, w11
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    sub w11, w11, #24
+; CHECK-GI-NEXT:    mov v2.b[14], w9
+; CHECK-GI-NEXT:    mov v3.b[14], w11
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w9, w10
+; CHECK-GI-NEXT:    sub w8, w8, #24
+; CHECK-GI-NEXT:    sub w9, w9, #24
+; CHECK-GI-NEXT:    mov v2.b[15], w8
+; CHECK-GI-NEXT:    mov v3.b[15], w9
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <32 x i8> @llvm.ctlz(<32 x i8> %d, i1 false)
+  ret <32 x i8> %s
+}
+
+define void @v2i16(ptr %p1) {
+; CHECK-SD-LABEL: v2i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldrh w8, [x0]
+; CHECK-SD-NEXT:    ldrh w9, [x0, #2]
+; CHECK-SD-NEXT:    movi v0.2s, #16
+; CHECK-SD-NEXT:    fmov s1, w8
+; CHECK-SD-NEXT:    mov v1.s[1], w9
+; CHECK-SD-NEXT:    clz v1.2s, v1.2s
+; CHECK-SD-NEXT:    sub v0.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    mov w8, v0.s[1]
+; CHECK-SD-NEXT:    fmov w9, s0
+; CHECK-SD-NEXT:    strh w9, [x0]
+; CHECK-SD-NEXT:    strh w8, [x0, #2]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrh w8, [x0]
+; CHECK-GI-NEXT:    ldrh w9, [x0, #2]
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    sub w8, w8, #16
+; CHECK-GI-NEXT:    sub w9, w9, #16
+; CHECK-GI-NEXT:    strh w8, [x0]
+; CHECK-GI-NEXT:    strh w9, [x0, #2]
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = load <2 x i16>, ptr %p1
+  %s = call <2 x i16> @llvm.ctlz(<2 x i16> %d, i1 false)
+  store <2 x i16> %s, ptr %p1
+  ret void
+}
+
+define void @v3i16(ptr %p1) {
+; CHECK-SD-LABEL: v3i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ldr d0, [x0]
+; CHECK-SD-NEXT:    add x8, x0, #4
+; CHECK-SD-NEXT:    clz v0.4h, v0.4h
+; CHECK-SD-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-SD-NEXT:    str s0, [x0]
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    ldrh w8, [x0]
+; CHECK-GI-NEXT:    ldrh w9, [x0, #2]
+; CHECK-GI-NEXT:    ldrh w10, [x0, #4]
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w10
+; CHECK-GI-NEXT:    sub w8, w8, #16
+; CHECK-GI-NEXT:    sub w9, w9, #16
+; CHECK-GI-NEXT:    strh w8, [x0]
+; CHECK-GI-NEXT:    sub w8, w10, #16
+; CHECK-GI-NEXT:    strh w9, [x0, #2]
+; CHECK-GI-NEXT:    strh w8, [x0, #4]
+; CHECK-GI-NEXT:    ret
+entry:
+  %d = load <3 x i16>, ptr %p1
+  %s = call <3 x i16> @llvm.ctlz(<3 x i16> %d, i1 false)
+  store <3 x i16> %s, ptr %p1
+  ret void
+}
+
+define <4 x i16> @v4i16(<4 x i16> %d) {
+; CHECK-LABEL: v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    clz v0.4h, v0.4h
+; CHECK-NEXT:    ret
+entry:
+  %s = call <4 x i16> @llvm.ctlz(<4 x i16> %d, i1 false)
+  ret <4 x i16> %s
+}
+
+define <8 x i16> @v8i16(<8 x i16> %d) {
+; CHECK-LABEL: v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    clz v0.8h, v0.8h
+; CHECK-NEXT:    ret
+entry:
+  %s = call <8 x i16> @llvm.ctlz(<8 x i16> %d, i1 false)
+  ret <8 x i16> %s
+}
+
+define <16 x i16> @v16i16(<16 x i16> %d) {
+; CHECK-SD-LABEL: v16i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    clz v0.8h, v0.8h
+; CHECK-SD-NEXT:    clz v1.8h, v1.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v16i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    umov w8, v0.h[0]
+; CHECK-GI-NEXT:    umov w10, v1.h[0]
+; CHECK-GI-NEXT:    umov w9, v0.h[1]
+; CHECK-GI-NEXT:    umov w11, v1.h[1]
+; CHECK-GI-NEXT:    umov w12, v0.h[2]
+; CHECK-GI-NEXT:    umov w13, v1.h[2]
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w10, w10
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    sub w8, w8, #16
+; CHECK-GI-NEXT:    sub w10, w10, #16
+; CHECK-GI-NEXT:    clz w11, w11
+; CHECK-GI-NEXT:    fmov s2, w8
+; CHECK-GI-NEXT:    fmov s3, w10
+; CHECK-GI-NEXT:    sub w9, w9, #16
+; CHECK-GI-NEXT:    sub w11, w11, #16
+; CHECK-GI-NEXT:    umov w8, v0.h[3]
+; CHECK-GI-NEXT:    clz w10, w13
+; CHECK-GI-NEXT:    sub w10, w10, #16
+; CHECK-GI-NEXT:    mov v2.h[1], w9
+; CHECK-GI-NEXT:    mov v3.h[1], w11
+; CHECK-GI-NEXT:    clz w9, w12
+; CHECK-GI-NEXT:    umov w11, v1.h[3]
+; CHECK-GI-NEXT:    sub w9, w9, #16
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    sub w8, w8, #16
+; CHECK-GI-NEXT:    mov v2.h[2], w9
+; CHECK-GI-NEXT:    mov v3.h[2], w10
+; CHECK-GI-NEXT:    umov w9, v0.h[4]
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.h[4]
+; CHECK-GI-NEXT:    sub w10, w10, #16
+; CHECK-GI-NEXT:    mov v2.h[3], w8
+; CHECK-GI-NEXT:    mov v3.h[3], w10
+; CHECK-GI-NEXT:    umov w8, v0.h[5]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.h[5]
+; CHECK-GI-NEXT:    sub w9, w9, #16
+; CHECK-GI-NEXT:    sub w10, w10, #16
+; CHECK-GI-NEXT:    mov v2.h[4], w9
+; CHECK-GI-NEXT:    mov v3.h[4], w10
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    umov w9, v0.h[6]
+; CHECK-GI-NEXT:    clz w10, w11
+; CHECK-GI-NEXT:    umov w11, v1.h[6]
+; CHECK-GI-NEXT:    sub w8, w8, #16
+; CHECK-GI-NEXT:    sub w10, w10, #16
+; CHECK-GI-NEXT:    mov v2.h[5], w8
+; CHECK-GI-NEXT:    mov v3.h[5], w10
+; CHECK-GI-NEXT:    umov w8, v0.h[7]
+; CHECK-GI-NEXT:    umov w10, v1.h[7]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w11, w11
+; CHECK-GI-NEXT:    sub w9, w9, #16
+; CHECK-GI-NEXT:    sub w11, w11, #16
+; CHECK-GI-NEXT:    mov v2.h[6], w9
+; CHECK-GI-NEXT:    mov v3.h[6], w11
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w9, w10
+; CHECK-GI-NEXT:    sub w8, w8, #16
+; CHECK-GI-NEXT:    sub w9, w9, #16
+; CHECK-GI-NEXT:    mov v2.h[7], w8
+; CHECK-GI-NEXT:    mov v3.h[7], w9
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <16 x i16> @llvm.ctlz(<16 x i16> %d, i1 false)
+  ret <16 x i16> %s
+}
+
+define <2 x i32> @v2i32(<2 x i32> %d) {
+; CHECK-LABEL: v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    clz v0.2s, v0.2s
+; CHECK-NEXT:    ret
+entry:
+  %s = call <2 x i32> @llvm.ctlz(<2 x i32> %d, i1 false)
+  ret <2 x i32> %s
+}
+
+define <3 x i32> @v3i32(<3 x i32> %d) {
+; CHECK-SD-LABEL: v3i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    clz v0.4s, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov w8, s0
+; CHECK-GI-NEXT:    mov w9, v0.s[1]
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    mov v1.s[0], w8
+; CHECK-GI-NEXT:    mov w8, v0.s[2]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    mov v1.s[1], w9
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    mov v1.s[2], w8
+; CHECK-GI-NEXT:    mov v0.16b, v1.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <3 x i32> @llvm.ctlz(<3 x i32> %d, i1 false)
+  ret <3 x i32> %s
+}
+
+define <4 x i32> @v4i32(<4 x i32> %d) {
+; CHECK-LABEL: v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    clz v0.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %s = call <4 x i32> @llvm.ctlz(<4 x i32> %d, i1 false)
+  ret <4 x i32> %s
+}
+
+define <8 x i32> @v8i32(<8 x i32> %d) {
+; CHECK-SD-LABEL: v8i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    clz v0.4s, v0.4s
+; CHECK-SD-NEXT:    clz v1.4s, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v8i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov w9, s0
+; CHECK-GI-NEXT:    fmov w10, s1
+; CHECK-GI-NEXT:    mov w8, v0.s[1]
+; CHECK-GI-NEXT:    mov w11, v1.s[1]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w10
+; CHECK-GI-NEXT:    mov v2.s[0], w9
+; CHECK-GI-NEXT:    mov v3.s[0], w10
+; CHECK-GI-NEXT:    mov w9, v0.s[2]
+; CHECK-GI-NEXT:    mov w10, v1.s[2]
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w11, w11
+; CHECK-GI-NEXT:    mov v2.s[1], w8
+; CHECK-GI-NEXT:    mov v3.s[1], w11
+; CHECK-GI-NEXT:    mov w8, v0.s[3]
+; CHECK-GI-NEXT:    mov w11, v1.s[3]
+; CHECK-GI-NEXT:    clz w9, w9
+; CHECK-GI-NEXT:    clz w10, w10
+; CHECK-GI-NEXT:    mov v2.s[2], w9
+; CHECK-GI-NEXT:    mov v3.s[2], w10
+; CHECK-GI-NEXT:    clz w8, w8
+; CHECK-GI-NEXT:    clz w9, w11
+; CHECK-GI-NEXT:    mov v2.s[3], w8
+; CHECK-GI-NEXT:    mov v3.s[3], w9
+; CHECK-GI-NEXT:    mov v0.16b, v2.16b
+; CHECK-GI-NEXT:    mov v1.16b, v3.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <8 x i32> @llvm.ctlz(<8 x i32> %d, i1 false)
+  ret <8 x i32> %s
+}
+
+define <2 x i64> @v2i64(<2 x i64> %d) {
+; CHECK-SD-LABEL: v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushr v1.2d, v0.2d, #1
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ushr v1.2d, v0.2d, #2
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ushr v1.2d, v0.2d, #4
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ushr v1.2d, v0.2d, #8
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ushr v1.2d, v0.2d, #16
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ushr v1.2d, v0.2d, #32
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    mvn v0.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-SD-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-SD-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    mov x9, v0.d[1]
+; CHECK-GI-NEXT:    clz x8, x8
+; CHECK-GI-NEXT:    mov v0.d[0], x8
+; CHECK-GI-NEXT:    clz x8, x9
+; CHECK-GI-NEXT:    mov v0.d[1], x8
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <2 x i64> @llvm.ctlz(<2 x i64> %d, i1 false)
+  ret <2 x i64> %s
+}
+
+define <3 x i64> @v3i64(<3 x i64> %d) {
+; CHECK-SD-LABEL: v3i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-SD-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-SD-NEXT:    ushr v1.2d, v0.2d, #1
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    ushr v1.2d, v2.2d, #1
+; CHECK-SD-NEXT:    ushr v3.2d, v0.2d, #2
+; CHECK-SD-NEXT:    orr v1.16b, v2.16b, v1.16b
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v1.2d, #2
+; CHECK-SD-NEXT:    ushr v3.2d, v0.2d, #4
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v1.2d, #4
+; CHECK-SD-NEXT:    ushr v3.2d, v0.2d, #8
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v1.2d, #8
+; CHECK-SD-NEXT:    ushr v3.2d, v0.2d, #16
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v1.2d, #16
+; CHECK-SD-NEXT:    ushr v3.2d, v0.2d, #32
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v1.2d, #32
+; CHECK-SD-NEXT:    mvn v0.16b, v0.16b
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v2.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    mvn v1.16b, v1.16b
+; CHECK-SD-NEXT:    cnt v1.16b, v1.16b
+; CHECK-SD-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-SD-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-SD-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-SD-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-SD-NEXT:    uaddlp v2.4s, v1.8h
+; CHECK-SD-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-SD-NEXT:    uaddlp v2.2d, v2.4s
+; CHECK-SD-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-SD-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-SD-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x8, d0
+; CHECK-GI-NEXT:    fmov x9, d1
+; CHECK-GI-NEXT:    fmov x10, d2
+; CHECK-GI-NEXT:    clz x8, x8
+; CHECK-GI-NEXT:    clz x9, x9
+; CHECK-GI-NEXT:    clz x10, x10
+; CHECK-GI-NEXT:    fmov d0, x8
+; CHECK-GI-NEXT:    fmov d1, x9
+; CHECK-GI-NEXT:    fmov d2, x10
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <3 x i64> @llvm.ctlz(<3 x i64> %d, i1 false)
+  ret <3 x i64> %s
+}
+
+define <4 x i64> @v4i64(<4 x i64> %d) {
+; CHECK-SD-LABEL: v4i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    ushr v2.2d, v0.2d, #1
+; CHECK-SD-NEXT:    ushr v3.2d, v1.2d, #1
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v0.2d, #2
+; CHECK-SD-NEXT:    ushr v3.2d, v1.2d, #2
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v0.2d, #4
+; CHECK-SD-NEXT:    ushr v3.2d, v1.2d, #4
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v0.2d, #8
+; CHECK-SD-NEXT:    ushr v3.2d, v1.2d, #8
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v0.2d, #16
+; CHECK-SD-NEXT:    ushr v3.2d, v1.2d, #16
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT:    ushr v2.2d, v0.2d, #32
+; CHECK-SD-NEXT:    ushr v3.2d, v1.2d, #32
+; CHECK-SD-NEXT:    orr v0.16b, v0.16b, v2.16b
+; CHECK-SD-NEXT:    orr v1.16b, v1.16b, v3.16b
+; CHECK-SD-NEXT:    mvn v0.16b, v0.16b
+; CHECK-SD-NEXT:    mvn v1.16b, v1.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v1.16b, v1.16b
+; CHECK-SD-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-SD-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-SD-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-SD-NEXT:    uaddlp v1.4s, v1.8h
+; CHECK-SD-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-SD-NEXT:    uaddlp v1.2d, v1.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    fmov x9, d0
+; CHECK-GI-NEXT:    fmov x10, d1
+; CHECK-GI-NEXT:    mov x8, v0.d[1]
+; CHECK-GI-NEXT:    mov x11, v1.d[1]
+; CHECK-GI-NEXT:    clz x9, x9
+; CHECK-GI-NEXT:    clz x10, x10
+; CHECK-GI-NEXT:    mov v0.d[0], x9
+; CHECK-GI-NEXT:    mov v1.d[0], x10
+; CHECK-GI-NEXT:    clz x8, x8
+; CHECK-GI-NEXT:    clz x9, x11
+; CHECK-GI-NEXT:    mov v0.d[1], x8
+; CHECK-GI-NEXT:    mov v1.d[1], x9
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <4 x i64> @llvm.ctlz(<4 x i64> %d, i1 false)
+  ret <4 x i64> %s
+}
+
+define <2 x i128> @v2i128(<2 x i128> %d) {
+; CHECK-SD-LABEL: v2i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    clz x8, x0
+; CHECK-SD-NEXT:    clz x9, x1
+; CHECK-SD-NEXT:    cmp x1, #0
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    clz x10, x2
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    csel x0, x9, x8, ne
+; CHECK-SD-NEXT:    clz x8, x3
+; CHECK-SD-NEXT:    add x9, x10, #64
+; CHECK-SD-NEXT:    cmp x3, #0
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    csel x2, x8, x9, ne
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, wzr
+; CHECK-GI-NEXT:    clz x9, x0
+; CHECK-GI-NEXT:    clz x11, x1
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    adc x10, xzr, xzr
+; CHECK-GI-NEXT:    cmp x1, #0
+; CHECK-GI-NEXT:    csel x0, x9, x11, eq
+; CHECK-GI-NEXT:    clz x9, x2
+; CHECK-GI-NEXT:    csel x1, x10, xzr, eq
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    clz x10, x3
+; CHECK-GI-NEXT:    adc x8, xzr, xzr
+; CHECK-GI-NEXT:    cmp x3, #0
+; CHECK-GI-NEXT:    csel x2, x9, x10, eq
+; CHECK-GI-NEXT:    csel x3, x8, xzr, eq
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <2 x i128> @llvm.ctlz(<2 x i128> %d, i1 false)
+  ret <2 x i128> %s
+}
+
+define <3 x i128> @v3i128(<3 x i128> %d) {
+; CHECK-SD-LABEL: v3i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    clz x8, x0
+; CHECK-SD-NEXT:    clz x9, x1
+; CHECK-SD-NEXT:    cmp x1, #0
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    clz x10, x2
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    csel x0, x9, x8, ne
+; CHECK-SD-NEXT:    clz x8, x3
+; CHECK-SD-NEXT:    add x9, x10, #64
+; CHECK-SD-NEXT:    cmp x3, #0
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    csel x2, x8, x9, ne
+; CHECK-SD-NEXT:    clz x8, x4
+; CHECK-SD-NEXT:    clz x9, x5
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    cmp x5, #0
+; CHECK-SD-NEXT:    mov x5, xzr
+; CHECK-SD-NEXT:    csel x4, x9, x8, ne
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v3i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, wzr
+; CHECK-GI-NEXT:    clz x9, x0
+; CHECK-GI-NEXT:    clz x11, x1
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    adc x10, xzr, xzr
+; CHECK-GI-NEXT:    cmp x1, #0
+; CHECK-GI-NEXT:    csel x0, x9, x11, eq
+; CHECK-GI-NEXT:    clz x9, x2
+; CHECK-GI-NEXT:    csel x1, x10, xzr, eq
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    clz x11, x3
+; CHECK-GI-NEXT:    adc x10, xzr, xzr
+; CHECK-GI-NEXT:    cmp x3, #0
+; CHECK-GI-NEXT:    csel x2, x9, x11, eq
+; CHECK-GI-NEXT:    clz x9, x4
+; CHECK-GI-NEXT:    csel x3, x10, xzr, eq
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    clz x10, x5
+; CHECK-GI-NEXT:    adc x8, xzr, xzr
+; CHECK-GI-NEXT:    cmp x5, #0
+; CHECK-GI-NEXT:    csel x4, x9, x10, eq
+; CHECK-GI-NEXT:    csel x5, x8, xzr, eq
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <3 x i128> @llvm.ctlz(<3 x i128> %d, i1 false)
+  ret <3 x i128> %s
+}
+
+define <4 x i128> @v4i128(<4 x i128> %d) {
+; CHECK-SD-LABEL: v4i128:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    clz x9, x0
+; CHECK-SD-NEXT:    clz x8, x2
+; CHECK-SD-NEXT:    clz x10, x1
+; CHECK-SD-NEXT:    add x9, x9, #64
+; CHECK-SD-NEXT:    cmp x1, #0
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    csel x0, x10, x9, ne
+; CHECK-SD-NEXT:    clz x9, x3
+; CHECK-SD-NEXT:    cmp x3, #0
+; CHECK-SD-NEXT:    csel x2, x9, x8, ne
+; CHECK-SD-NEXT:    clz x8, x4
+; CHECK-SD-NEXT:    clz x9, x5
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    cmp x5, #0
+; CHECK-SD-NEXT:    mov x1, xzr
+; CHECK-SD-NEXT:    csel x4, x9, x8, ne
+; CHECK-SD-NEXT:    clz x8, x6
+; CHECK-SD-NEXT:    clz x9, x7
+; CHECK-SD-NEXT:    add x8, x8, #64
+; CHECK-SD-NEXT:    cmp x7, #0
+; CHECK-SD-NEXT:    mov x3, xzr
+; CHECK-SD-NEXT:    csel x6, x9, x8, ne
+; CHECK-SD-NEXT:    mov x5, xzr
+; CHECK-SD-NEXT:    mov x7, xzr
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i128:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    mov w8, wzr
+; CHECK-GI-NEXT:    clz x9, x0
+; CHECK-GI-NEXT:    clz x11, x1
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    adc x10, xzr, xzr
+; CHECK-GI-NEXT:    cmp x1, #0
+; CHECK-GI-NEXT:    csel x0, x9, x11, eq
+; CHECK-GI-NEXT:    clz x9, x2
+; CHECK-GI-NEXT:    csel x1, x10, xzr, eq
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    clz x11, x3
+; CHECK-GI-NEXT:    adc x10, xzr, xzr
+; CHECK-GI-NEXT:    cmp x3, #0
+; CHECK-GI-NEXT:    csel x2, x9, x11, eq
+; CHECK-GI-NEXT:    clz x9, x4
+; CHECK-GI-NEXT:    csel x3, x10, xzr, eq
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    clz x11, x5
+; CHECK-GI-NEXT:    adc x10, xzr, xzr
+; CHECK-GI-NEXT:    cmp x5, #0
+; CHECK-GI-NEXT:    csel x4, x9, x11, eq
+; CHECK-GI-NEXT:    clz x9, x6
+; CHECK-GI-NEXT:    csel x5, x10, xzr, eq
+; CHECK-GI-NEXT:    cmp w8, #1
+; CHECK-GI-NEXT:    add x9, x9, #64
+; CHECK-GI-NEXT:    clz x10, x7
+; CHECK-GI-NEXT:    adc x8, xzr, xzr
+; CHECK-GI-NEXT:    cmp x7, #0
+; CHECK-GI-NEXT:    csel x6, x9, x10, eq
+; CHECK-GI-NEXT:    csel x7, x8, xzr, eq
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <4 x i128> @llvm.ctlz(<4 x i128> %d, i1 false)
+  ret <4 x i128> %s
+}
diff --git a/llvm/test/CodeGen/AArch64/ctpop.ll b/llvm/test/CodeGen/AArch64/ctpop.ll
new file mode 100644
index 000000000000..785a447123b5
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/ctpop.ll
@@ -0,0 +1,377 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:       warning: Instruction selection used fallback path for v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v32i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v2i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v16i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v8i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v2i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i128
+
+define void @v2i8(ptr %p1) {
+; CHECK-LABEL: v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrb w8, [x0]
+; CHECK-NEXT:    ldrb w9, [x0, #1]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    strb w9, [x0]
+; CHECK-NEXT:    strb w8, [x0, #1]
+; CHECK-NEXT:    ret
+entry:
+  %d = load <2 x i8>, ptr %p1
+  %s = call <2 x i8> @llvm.ctpop(<2 x i8> %d)
+  store <2 x i8> %s, ptr %p1
+  ret void
+}
+
+define void @v3i8(ptr %p1) {
+; CHECK-LABEL: v3i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    zip1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT:    bic v0.4h, #255, lsl #8
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
+; CHECK-NEXT:    umov w8, v0.h[2]
+; CHECK-NEXT:    str s1, [sp, #12]
+; CHECK-NEXT:    ldrh w9, [sp, #12]
+; CHECK-NEXT:    strb w8, [x0, #2]
+; CHECK-NEXT:    strh w9, [x0]
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+entry:
+  %d = load <3 x i8>, ptr %p1
+  %s = call <3 x i8> @llvm.ctpop(<3 x i8> %d)
+  store <3 x i8> %s, ptr %p1
+  ret void
+}
+
+define void @v4i8(ptr %p1) {
+; CHECK-LABEL: v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %d = load <4 x i8>, ptr %p1
+  %s = call <4 x i8> @llvm.ctpop(<4 x i8> %d)
+  store <4 x i8> %s, ptr %p1
+  ret void
+}
+
+define <8 x i8> @v8i8(<8 x i8> %d) {
+; CHECK-LABEL: v8i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <8 x i8> @llvm.ctpop(<8 x i8> %d)
+  ret <8 x i8> %s
+}
+
+define <16 x i8> @v16i8(<16 x i8> %d) {
+; CHECK-LABEL: v16i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <16 x i8> @llvm.ctpop(<16 x i8> %d)
+  ret <16 x i8> %s
+}
+
+define <32 x i8> @v32i8(<32 x i8> %d) {
+; CHECK-LABEL: v32i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <32 x i8> @llvm.ctpop(<32 x i8> %d)
+  ret <32 x i8> %s
+}
+
+define void @v2i16(ptr %p1) {
+; CHECK-LABEL: v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldrh w8, [x0]
+; CHECK-NEXT:    ldrh w9, [x0, #2]
+; CHECK-NEXT:    fmov s0, w8
+; CHECK-NEXT:    mov v0.s[1], w9
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    strh w9, [x0]
+; CHECK-NEXT:    strh w8, [x0, #2]
+; CHECK-NEXT:    ret
+entry:
+  %d = load <2 x i16>, ptr %p1
+  %s = call <2 x i16> @llvm.ctpop(<2 x i16> %d)
+  store <2 x i16> %s, ptr %p1
+  ret void
+}
+
+define void @v3i16(ptr %p1) {
+; CHECK-LABEL: v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr d0, [x0]
+; CHECK-NEXT:    add x8, x0, #4
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %d = load <3 x i16>, ptr %p1
+  %s = call <3 x i16> @llvm.ctpop(<3 x i16> %d)
+  store <3 x i16> %s, ptr %p1
+  ret void
+}
+
+define <4 x i16> @v4i16(<4 x i16> %d) {
+; CHECK-LABEL: v4i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <4 x i16> @llvm.ctpop(<4 x i16> %d)
+  ret <4 x i16> %s
+}
+
+define <8 x i16> @v8i16(<8 x i16> %d) {
+; CHECK-LABEL: v8i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <8 x i16> @llvm.ctpop(<8 x i16> %d)
+  ret <8 x i16> %s
+}
+
+define <16 x i16> @v16i16(<16 x i16> %d) {
+; CHECK-LABEL: v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <16 x i16> @llvm.ctpop(<16 x i16> %d)
+  ret <16 x i16> %s
+}
+
+define <2 x i32> @v2i32(<2 x i32> %d) {
+; CHECK-LABEL: v2i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.8b, v0.8b
+; CHECK-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-NEXT:    ret
+entry:
+  %s = call <2 x i32> @llvm.ctpop(<2 x i32> %d)
+  ret <2 x i32> %s
+}
+
+define <3 x i32> @v3i32(<3 x i32> %d) {
+; CHECK-LABEL: v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    ret
+entry:
+  %s = call <3 x i32> @llvm.ctpop(<3 x i32> %d)
+  ret <3 x i32> %s
+}
+
+define <4 x i32> @v4i32(<4 x i32> %d) {
+; CHECK-LABEL: v4i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    ret
+entry:
+  %s = call <4 x i32> @llvm.ctpop(<4 x i32> %d)
+  ret <4 x i32> %s
+}
+
+define <8 x i32> @v8i32(<8 x i32> %d) {
+; CHECK-LABEL: v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    uaddlp v1.4s, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %s = call <8 x i32> @llvm.ctpop(<8 x i32> %d)
+  ret <8 x i32> %s
+}
+
+define <2 x i64> @v2i64(<2 x i64> %d) {
+; CHECK-LABEL: v2i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %s = call <2 x i64> @llvm.ctpop(<2 x i64> %d)
+  ret <2 x i64> %s
+}
+
+define <3 x i64> @v3i64(<3 x i64> %d) {
+; CHECK-LABEL: v3i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    cnt v1.16b, v2.16b
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-NEXT:    uaddlp v2.4s, v1.8h
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v2.2d, v2.4s
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    ret
+entry:
+  %s = call <3 x i64> @llvm.ctpop(<3 x i64> %d)
+  ret <3 x i64> %s
+}
+
+define <4 x i64> @v4i64(<4 x i64> %d) {
+; CHECK-LABEL: v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    uaddlp v1.4s, v1.8h
+; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    uaddlp v1.2d, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %s = call <4 x i64> @llvm.ctpop(<4 x i64> %d)
+  ret <4 x i64> %s
+}
+
+define <2 x i128> @v2i128(<2 x i128> %d) {
+; CHECK-LABEL: v2i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d0, x2
+; CHECK-NEXT:    fmov d1, x0
+; CHECK-NEXT:    mov v1.d[1], x1
+; CHECK-NEXT:    mov v0.d[1], x3
+; CHECK-NEXT:    mov x1, xzr
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    addv b1, v1.16b
+; CHECK-NEXT:    addv b0, v0.16b
+; CHECK-NEXT:    fmov x0, d1
+; CHECK-NEXT:    fmov x2, d0
+; CHECK-NEXT:    ret
+entry:
+  %s = call <2 x i128> @llvm.ctpop(<2 x i128> %d)
+  ret <2 x i128> %s
+}
+
+define <3 x i128> @v3i128(<3 x i128> %d) {
+; CHECK-LABEL: v3i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d0, x4
+; CHECK-NEXT:    fmov d1, x2
+; CHECK-NEXT:    fmov d2, x0
+; CHECK-NEXT:    mov v0.d[1], x5
+; CHECK-NEXT:    mov v1.d[1], x3
+; CHECK-NEXT:    mov v2.d[1], x1
+; CHECK-NEXT:    mov x1, xzr
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    mov x5, xzr
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    cnt v2.16b, v2.16b
+; CHECK-NEXT:    addv b0, v0.16b
+; CHECK-NEXT:    addv b1, v1.16b
+; CHECK-NEXT:    addv b2, v2.16b
+; CHECK-NEXT:    fmov x0, d2
+; CHECK-NEXT:    fmov x2, d1
+; CHECK-NEXT:    fmov x4, d0
+; CHECK-NEXT:    ret
+entry:
+  %s = call <3 x i128> @llvm.ctpop(<3 x i128> %d)
+  ret <3 x i128> %s
+}
+
+define <4 x i128> @v4i128(<4 x i128> %d) {
+; CHECK-LABEL: v4i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    fmov d0, x6
+; CHECK-NEXT:    fmov d1, x4
+; CHECK-NEXT:    fmov d2, x2
+; CHECK-NEXT:    fmov d3, x0
+; CHECK-NEXT:    mov v1.d[1], x5
+; CHECK-NEXT:    mov v2.d[1], x3
+; CHECK-NEXT:    mov v0.d[1], x7
+; CHECK-NEXT:    mov v3.d[1], x1
+; CHECK-NEXT:    mov x1, xzr
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    mov x5, xzr
+; CHECK-NEXT:    mov x7, xzr
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    cnt v2.16b, v2.16b
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v3.16b, v3.16b
+; CHECK-NEXT:    addv b1, v1.16b
+; CHECK-NEXT:    addv b2, v2.16b
+; CHECK-NEXT:    addv b0, v0.16b
+; CHECK-NEXT:    addv b3, v3.16b
+; CHECK-NEXT:    fmov x2, d2
+; CHECK-NEXT:    fmov x4, d1
+; CHECK-NEXT:    fmov x6, d0
+; CHECK-NEXT:    fmov x0, d3
+; CHECK-NEXT:    ret
+entry:
+  %s = call <4 x i128> @llvm.ctpop(<4 x i128> %d)
+  ret <4 x i128> %s
+}
+;; NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:
+; CHECK-GI: {{.*}}
+; CHECK-SD: {{.*}}
diff --git a/llvm/test/CodeGen/AArch64/cttz.ll b/llvm/test/CodeGen/AArch64/cttz.ll
new file mode 100644
index 000000000000..a254df229c12
--- /dev/null
+++ b/llvm/test/CodeGen/AArch64/cttz.ll
@@ -0,0 +1,530 @@
+; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py UTC_ARGS: --version 5
+; RUN: llc -mtriple=aarch64-none-eabi -verify-machineinstrs %s -o - | FileCheck %s --check-prefixes=CHECK,CHECK-SD
+; RUN: llc -mtriple=aarch64-none-eabi -global-isel -global-isel-abort=2 -verify-machineinstrs %s -o - 2>&1 | FileCheck %s --check-prefixes=CHECK,CHECK-GI
+
+; CHECK-GI:       warning: Instruction selection used fallback path for v2i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v32i8
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v2i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v16i16
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v8i32
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i64
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v2i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v3i128
+; CHECK-GI-NEXT:  warning: Instruction selection used fallback path for v4i128
+
+define void @v2i8(ptr %p1) {
+; CHECK-LABEL: v2i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1 { v0.b }[0], [x0]
+; CHECK-NEXT:    add x8, x0, #1
+; CHECK-NEXT:    movi v1.2s, #1
+; CHECK-NEXT:    ld1 { v0.b }[4], [x8]
+; CHECK-NEXT:    orr v0.2s, #1, lsl #8
+; CHECK-NEXT:    sub v1.2s, v0.2s, v1.2s
+; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    movi v1.2s, #32
+; CHECK-NEXT:    clz v0.2s, v0.2s
+; CHECK-NEXT:    sub v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    strb w9, [x0]
+; CHECK-NEXT:    strb w8, [x0, #1]
+; CHECK-NEXT:    ret
+entry:
+  %d = load <2 x i8>, ptr %p1
+  %s = call <2 x i8> @llvm.cttz(<2 x i8> %d, i1 false)
+  store <2 x i8> %s, ptr %p1
+  ret void
+}
+
+define void @v3i8(ptr %p1) {
+; CHECK-LABEL: v3i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    sub sp, sp, #16
+; CHECK-NEXT:    .cfi_def_cfa_offset 16
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    movi v1.4h, #1
+; CHECK-NEXT:    zip1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT:    orr v0.4h, #1, lsl #8
+; CHECK-NEXT:    sub v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    movi v1.4h, #16
+; CHECK-NEXT:    clz v0.4h, v0.4h
+; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    uzp1 v1.8b, v0.8b, v0.8b
+; CHECK-NEXT:    umov w8, v0.h[2]
+; CHECK-NEXT:    str s1, [sp, #12]
+; CHECK-NEXT:    ldrh w9, [sp, #12]
+; CHECK-NEXT:    strb w8, [x0, #2]
+; CHECK-NEXT:    strh w9, [x0]
+; CHECK-NEXT:    add sp, sp, #16
+; CHECK-NEXT:    ret
+entry:
+  %d = load <3 x i8>, ptr %p1
+  %s = call <3 x i8> @llvm.cttz(<3 x i8> %d, i1 false)
+  store <3 x i8> %s, ptr %p1
+  ret void
+}
+
+define void @v4i8(ptr %p1) {
+; CHECK-LABEL: v4i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ldr s0, [x0]
+; CHECK-NEXT:    movi v1.4h, #1
+; CHECK-NEXT:    ushll v0.8h, v0.8b, #0
+; CHECK-NEXT:    orr v0.4h, #1, lsl #8
+; CHECK-NEXT:    sub v1.4h, v0.4h, v1.4h
+; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    movi v1.4h, #16
+; CHECK-NEXT:    clz v0.4h, v0.4h
+; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    uzp1 v0.8b, v0.8b, v0.8b
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %d = load <4 x i8>, ptr %p1
+  %s = call <4 x i8> @llvm.cttz(<4 x i8> %d, i1 false)
+  store <4 x i8> %s, ptr %p1
+  ret void
+}
+
+define <8 x i8> @v8i8(<8 x i8> %d) {
+; CHECK-SD-LABEL: v8i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8b, #1
+; CHECK-SD-NEXT:    sub v1.8b, v0.8b, v1.8b
+; CHECK-SD-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    cnt v0.8b, v0.8b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v8i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi d1, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v1.8b, v0.8b, v1.8b
+; CHECK-GI-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <8 x i8> @llvm.cttz(<8 x i8> %d, i1 false)
+  ret <8 x i8> %s
+}
+
+define <16 x i8> @v16i8(<16 x i8> %d) {
+; CHECK-SD-LABEL: v16i8:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.16b, #1
+; CHECK-SD-NEXT:    sub v1.16b, v0.16b, v1.16b
+; CHECK-SD-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v16i8:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v1.16b, v0.16b, v1.16b
+; CHECK-GI-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <16 x i8> @llvm.cttz(<16 x i8> %d, i1 false)
+  ret <16 x i8> %s
+}
+
+define <32 x i8> @v32i8(<32 x i8> %d) {
+; CHECK-LABEL: v32i8:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v2.16b, #1
+; CHECK-NEXT:    sub v3.16b, v0.16b, v2.16b
+; CHECK-NEXT:    sub v2.16b, v1.16b, v2.16b
+; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    ret
+entry:
+  %s = call <32 x i8> @llvm.cttz(<32 x i8> %d, i1 false)
+  ret <32 x i8> %s
+}
+
+define void @v2i16(ptr %p1) {
+; CHECK-LABEL: v2i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    ld1 { v0.h }[0], [x0]
+; CHECK-NEXT:    add x8, x0, #2
+; CHECK-NEXT:    movi v1.2s, #1
+; CHECK-NEXT:    ld1 { v0.h }[2], [x8]
+; CHECK-NEXT:    orr v0.2s, #1, lsl #16
+; CHECK-NEXT:    sub v1.2s, v0.2s, v1.2s
+; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-NEXT:    movi v1.2s, #32
+; CHECK-NEXT:    clz v0.2s, v0.2s
+; CHECK-NEXT:    sub v0.2s, v1.2s, v0.2s
+; CHECK-NEXT:    mov w8, v0.s[1]
+; CHECK-NEXT:    fmov w9, s0
+; CHECK-NEXT:    strh w9, [x0]
+; CHECK-NEXT:    strh w8, [x0, #2]
+; CHECK-NEXT:    ret
+entry:
+  %d = load <2 x i16>, ptr %p1
+  %s = call <2 x i16> @llvm.cttz(<2 x i16> %d, i1 false)
+  store <2 x i16> %s, ptr %p1
+  ret void
+}
+
+define void @v3i16(ptr %p1) {
+; CHECK-LABEL: v3i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v0.4h, #1
+; CHECK-NEXT:    ldr d1, [x0]
+; CHECK-NEXT:    add x8, x0, #4
+; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    bic v0.8b, v0.8b, v1.8b
+; CHECK-NEXT:    movi v1.4h, #16
+; CHECK-NEXT:    clz v0.4h, v0.4h
+; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-NEXT:    st1 { v0.h }[2], [x8]
+; CHECK-NEXT:    str s0, [x0]
+; CHECK-NEXT:    ret
+entry:
+  %d = load <3 x i16>, ptr %p1
+  %s = call <3 x i16> @llvm.cttz(<3 x i16> %d, i1 false)
+  store <3 x i16> %s, ptr %p1
+  ret void
+}
+
+define <4 x i16> @v4i16(<4 x i16> %d) {
+; CHECK-SD-LABEL: v4i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.4h, #1
+; CHECK-SD-NEXT:    sub v1.4h, v0.4h, v1.4h
+; CHECK-SD-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    movi v1.4h, #16
+; CHECK-SD-NEXT:    clz v0.4h, v0.4h
+; CHECK-SD-NEXT:    sub v0.4h, v1.4h, v0.4h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi d1, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v1.4h, v0.4h, v1.4h
+; CHECK-GI-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <4 x i16> @llvm.cttz(<4 x i16> %d, i1 false)
+  ret <4 x i16> %s
+}
+
+define <8 x i16> @v8i16(<8 x i16> %d) {
+; CHECK-SD-LABEL: v8i16:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.8h, #1
+; CHECK-SD-NEXT:    sub v1.8h, v0.8h, v1.8h
+; CHECK-SD-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    movi v1.8h, #16
+; CHECK-SD-NEXT:    clz v0.8h, v0.8h
+; CHECK-SD-NEXT:    sub v0.8h, v1.8h, v0.8h
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v8i16:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v1.8h, v0.8h, v1.8h
+; CHECK-GI-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <8 x i16> @llvm.cttz(<8 x i16> %d, i1 false)
+  ret <8 x i16> %s
+}
+
+define <16 x i16> @v16i16(<16 x i16> %d) {
+; CHECK-LABEL: v16i16:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v2.8h, #1
+; CHECK-NEXT:    sub v3.8h, v0.8h, v2.8h
+; CHECK-NEXT:    sub v2.8h, v1.8h, v2.8h
+; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-NEXT:    movi v2.8h, #16
+; CHECK-NEXT:    clz v0.8h, v0.8h
+; CHECK-NEXT:    clz v1.8h, v1.8h
+; CHECK-NEXT:    sub v0.8h, v2.8h, v0.8h
+; CHECK-NEXT:    sub v1.8h, v2.8h, v1.8h
+; CHECK-NEXT:    ret
+entry:
+  %s = call <16 x i16> @llvm.cttz(<16 x i16> %d, i1 false)
+  ret <16 x i16> %s
+}
+
+define <2 x i32> @v2i32(<2 x i32> %d) {
+; CHECK-SD-LABEL: v2i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.2s, #1
+; CHECK-SD-NEXT:    sub v1.2s, v0.2s, v1.2s
+; CHECK-SD-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-SD-NEXT:    movi v1.2s, #32
+; CHECK-SD-NEXT:    clz v0.2s, v0.2s
+; CHECK-SD-NEXT:    sub v0.2s, v1.2s, v0.2s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi d1, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v1.2s, v0.2s, v1.2s
+; CHECK-GI-NEXT:    bic v0.8b, v1.8b, v0.8b
+; CHECK-GI-NEXT:    cnt v0.8b, v0.8b
+; CHECK-GI-NEXT:    uaddlp v0.4h, v0.8b
+; CHECK-GI-NEXT:    uaddlp v0.2s, v0.4h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <2 x i32> @llvm.cttz(<2 x i32> %d, i1 false)
+  ret <2 x i32> %s
+}
+
+define <3 x i32> @v3i32(<3 x i32> %d) {
+; CHECK-LABEL: v3i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v1.4s, #1
+; CHECK-NEXT:    sub v1.4s, v0.4s, v1.4s
+; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-NEXT:    movi v1.4s, #32
+; CHECK-NEXT:    clz v0.4s, v0.4s
+; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
+; CHECK-NEXT:    ret
+entry:
+  %s = call <3 x i32> @llvm.cttz(<3 x i32> %d, i1 false)
+  ret <3 x i32> %s
+}
+
+define <4 x i32> @v4i32(<4 x i32> %d) {
+; CHECK-SD-LABEL: v4i32:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    movi v1.4s, #1
+; CHECK-SD-NEXT:    sub v1.4s, v0.4s, v1.4s
+; CHECK-SD-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    movi v1.4s, #32
+; CHECK-SD-NEXT:    clz v0.4s, v0.4s
+; CHECK-SD-NEXT:    sub v0.4s, v1.4s, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v4i32:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v1.4s, v0.4s, v1.4s
+; CHECK-GI-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <4 x i32> @llvm.cttz(<4 x i32> %d, i1 false)
+  ret <4 x i32> %s
+}
+
+define <8 x i32> @v8i32(<8 x i32> %d) {
+; CHECK-LABEL: v8i32:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    movi v2.4s, #1
+; CHECK-NEXT:    sub v3.4s, v0.4s, v2.4s
+; CHECK-NEXT:    sub v2.4s, v1.4s, v2.4s
+; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-NEXT:    movi v2.4s, #32
+; CHECK-NEXT:    clz v0.4s, v0.4s
+; CHECK-NEXT:    clz v1.4s, v1.4s
+; CHECK-NEXT:    sub v0.4s, v2.4s, v0.4s
+; CHECK-NEXT:    sub v1.4s, v2.4s, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %s = call <8 x i32> @llvm.cttz(<8 x i32> %d, i1 false)
+  ret <8 x i32> %s
+}
+
+define <2 x i64> @v2i64(<2 x i64> %d) {
+; CHECK-SD-LABEL: v2i64:
+; CHECK-SD:       // %bb.0: // %entry
+; CHECK-SD-NEXT:    mov w8, #1 // =0x1
+; CHECK-SD-NEXT:    dup v1.2d, x8
+; CHECK-SD-NEXT:    sub v1.2d, v0.2d, v1.2d
+; CHECK-SD-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-SD-NEXT:    cnt v0.16b, v0.16b
+; CHECK-SD-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-SD-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-SD-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-SD-NEXT:    ret
+;
+; CHECK-GI-LABEL: v2i64:
+; CHECK-GI:       // %bb.0: // %entry
+; CHECK-GI-NEXT:    movi v1.2d, #0xffffffffffffffff
+; CHECK-GI-NEXT:    add v1.2d, v0.2d, v1.2d
+; CHECK-GI-NEXT:    bic v0.16b, v1.16b, v0.16b
+; CHECK-GI-NEXT:    cnt v0.16b, v0.16b
+; CHECK-GI-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-GI-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-GI-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-GI-NEXT:    ret
+entry:
+  %s = call <2 x i64> @llvm.cttz(<2 x i64> %d, i1 false)
+  ret <2 x i64> %s
+}
+
+define <3 x i64> @v3i64(<3 x i64> %d) {
+; CHECK-LABEL: v3i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    // kill: def $d0 killed $d0 def $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 def $q1
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:    // kill: def $d2 killed $d2 def $q2
+; CHECK-NEXT:    mov v0.d[1], v1.d[0]
+; CHECK-NEXT:    dup v1.2d, x8
+; CHECK-NEXT:    sub v3.2d, v0.2d, v1.2d
+; CHECK-NEXT:    sub v1.2d, v2.2d, v1.2d
+; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bic v1.16b, v1.16b, v2.16b
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    uaddlp v2.4s, v1.8h
+; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    uaddlp v2.2d, v2.4s
+; CHECK-NEXT:    // kill: def $d2 killed $d2 killed $q2
+; CHECK-NEXT:    ext v1.16b, v0.16b, v0.16b, #8
+; CHECK-NEXT:    // kill: def $d0 killed $d0 killed $q0
+; CHECK-NEXT:    // kill: def $d1 killed $d1 killed $q1
+; CHECK-NEXT:    ret
+entry:
+  %s = call <3 x i64> @llvm.cttz(<3 x i64> %d, i1 false)
+  ret <3 x i64> %s
+}
+
+define <4 x i64> @v4i64(<4 x i64> %d) {
+; CHECK-LABEL: v4i64:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    mov w8, #1 // =0x1
+; CHECK-NEXT:    dup v2.2d, x8
+; CHECK-NEXT:    sub v3.2d, v0.2d, v2.2d
+; CHECK-NEXT:    sub v2.2d, v1.2d, v2.2d
+; CHECK-NEXT:    bic v0.16b, v3.16b, v0.16b
+; CHECK-NEXT:    bic v1.16b, v2.16b, v1.16b
+; CHECK-NEXT:    cnt v0.16b, v0.16b
+; CHECK-NEXT:    cnt v1.16b, v1.16b
+; CHECK-NEXT:    uaddlp v0.8h, v0.16b
+; CHECK-NEXT:    uaddlp v1.8h, v1.16b
+; CHECK-NEXT:    uaddlp v0.4s, v0.8h
+; CHECK-NEXT:    uaddlp v1.4s, v1.8h
+; CHECK-NEXT:    uaddlp v0.2d, v0.4s
+; CHECK-NEXT:    uaddlp v1.2d, v1.4s
+; CHECK-NEXT:    ret
+entry:
+  %s = call <4 x i64> @llvm.cttz(<4 x i64> %d, i1 false)
+  ret <4 x i64> %s
+}
+
+define <2 x i128> @v2i128(<2 x i128> %d) {
+; CHECK-LABEL: v2i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit x8, x1
+; CHECK-NEXT:    rbit x9, x0
+; CHECK-NEXT:    rbit x10, x3
+; CHECK-NEXT:    rbit x11, x2
+; CHECK-NEXT:    cmp x0, #0
+; CHECK-NEXT:    mov x1, xzr
+; CHECK-NEXT:    clz x8, x8
+; CHECK-NEXT:    clz x9, x9
+; CHECK-NEXT:    clz x10, x10
+; CHECK-NEXT:    add x8, x8, #64
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    csel x0, x9, x8, ne
+; CHECK-NEXT:    clz x8, x11
+; CHECK-NEXT:    add x9, x10, #64
+; CHECK-NEXT:    cmp x2, #0
+; CHECK-NEXT:    csel x2, x8, x9, ne
+; CHECK-NEXT:    ret
+entry:
+  %s = call <2 x i128> @llvm.cttz(<2 x i128> %d, i1 false)
+  ret <2 x i128> %s
+}
+
+define <3 x i128> @v3i128(<3 x i128> %d) {
+; CHECK-LABEL: v3i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit x8, x1
+; CHECK-NEXT:    rbit x9, x0
+; CHECK-NEXT:    rbit x11, x3
+; CHECK-NEXT:    rbit x10, x2
+; CHECK-NEXT:    cmp x0, #0
+; CHECK-NEXT:    rbit x12, x5
+; CHECK-NEXT:    clz x8, x8
+; CHECK-NEXT:    clz x9, x9
+; CHECK-NEXT:    clz x11, x11
+; CHECK-NEXT:    add x8, x8, #64
+; CHECK-NEXT:    clz x10, x10
+; CHECK-NEXT:    mov x1, xzr
+; CHECK-NEXT:    csel x0, x9, x8, ne
+; CHECK-NEXT:    add x8, x11, #64
+; CHECK-NEXT:    cmp x2, #0
+; CHECK-NEXT:    rbit x9, x4
+; CHECK-NEXT:    csel x2, x10, x8, ne
+; CHECK-NEXT:    clz x8, x12
+; CHECK-NEXT:    add x8, x8, #64
+; CHECK-NEXT:    cmp x4, #0
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    clz x9, x9
+; CHECK-NEXT:    mov x5, xzr
+; CHECK-NEXT:    csel x4, x9, x8, ne
+; CHECK-NEXT:    ret
+entry:
+  %s = call <3 x i128> @llvm.cttz(<3 x i128> %d, i1 false)
+  ret <3 x i128> %s
+}
+
+define <4 x i128> @v4i128(<4 x i128> %d) {
+; CHECK-LABEL: v4i128:
+; CHECK:       // %bb.0: // %entry
+; CHECK-NEXT:    rbit x9, x1
+; CHECK-NEXT:    rbit x10, x0
+; CHECK-NEXT:    rbit x8, x3
+; CHECK-NEXT:    rbit x11, x2
+; CHECK-NEXT:    cmp x0, #0
+; CHECK-NEXT:    mov x1, xzr
+; CHECK-NEXT:    clz x9, x9
+; CHECK-NEXT:    clz x10, x10
+; CHECK-NEXT:    clz x8, x8
+; CHECK-NEXT:    add x9, x9, #64
+; CHECK-NEXT:    add x8, x8, #64
+; CHECK-NEXT:    mov x3, xzr
+; CHECK-NEXT:    csel x0, x10, x9, ne
+; CHECK-NEXT:    clz x9, x11
+; CHECK-NEXT:    rbit x10, x4
+; CHECK-NEXT:    rbit x11, x5
+; CHECK-NEXT:    cmp x2, #0
+; CHECK-NEXT:    mov x5, xzr
+; CHECK-NEXT:    csel x2, x9, x8, ne
+; CHECK-NEXT:    clz x8, x10
+; CHECK-NEXT:    rbit x10, x7
+; CHECK-NEXT:    clz x9, x11
+; CHECK-NEXT:    cmp x4, #0
+; CHECK-NEXT:    rbit x11, x6
+; CHECK-NEXT:    add x9, x9, #64
+; CHECK-NEXT:    mov x7, xzr
+; CHECK-NEXT:    csel x4, x8, x9, ne
+; CHECK-NEXT:    clz x8, x10
+; CHECK-NEXT:    clz x9, x11
+; CHECK-NEXT:    add x8, x8, #64
+; CHECK-NEXT:    cmp x6, #0
+; CHECK-NEXT:    csel x6, x9, x8, ne
+; CHECK-NEXT:    ret
+entry:
+  %s = call <4 x i128> @llvm.cttz(<4 x i128> %d, i1 false)
+  ret <4 x i128> %s
+}
diff --git a/llvm/test/CodeGen/AArch64/vec_cttz.ll b/llvm/test/CodeGen/AArch64/vec_cttz.ll
deleted file mode 100644
index 26565f4ab095..000000000000
--- a/llvm/test/CodeGen/AArch64/vec_cttz.ll
+++ /dev/null
@@ -1,124 +0,0 @@
-; NOTE: Assertions have been autogenerated by utils/update_llc_test_checks.py
-; RUN: llc < %s -mtriple=aarch64-none-linux-gnu -mattr=+neon | FileCheck %s --check-prefix=CHECK
-
-declare <8 x i8> @llvm.cttz.v8i8(<8 x i8>, i1)
-declare <4 x i16> @llvm.cttz.v4i16(<4 x i16>, i1)
-declare <2 x i32> @llvm.cttz.v2i32(<2 x i32>, i1)
-declare <1 x i64> @llvm.cttz.v1i64(<1 x i64>, i1)
-
-declare <16 x i8> @llvm.cttz.v16i8(<16 x i8>, i1)
-declare <8 x i16> @llvm.cttz.v8i16(<8 x i16>, i1)
-declare <4 x i32> @llvm.cttz.v4i32(<4 x i32>, i1)
-declare <2 x i64> @llvm.cttz.v2i64(<2 x i64>, i1)
-
-define <8 x i8> @cttz_v8i8(<8 x i8> %a) nounwind {
-; CHECK-LABEL: cttz_v8i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.8b, #1
-; CHECK-NEXT:    sub v1.8b, v0.8b, v1.8b
-; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    ret
-    %b = call <8 x i8> @llvm.cttz.v8i8(<8 x i8> %a, i1 true)
-    ret <8 x i8> %b
-}
-
-define <4 x i16> @cttz_v4i16(<4 x i16> %a) nounwind {
-; CHECK-LABEL: cttz_v4i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.4h, #1
-; CHECK-NEXT:    sub v1.4h, v0.4h, v1.4h
-; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    movi v1.4h, #16
-; CHECK-NEXT:    clz v0.4h, v0.4h
-; CHECK-NEXT:    sub v0.4h, v1.4h, v0.4h
-; CHECK-NEXT:    ret
-    %b = call <4 x i16> @llvm.cttz.v4i16(<4 x i16> %a, i1 true)
-    ret <4 x i16> %b
-}
-
-define <2 x i32> @cttz_v2i32(<2 x i32> %a) nounwind {
-; CHECK-LABEL: cttz_v2i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.2s, #1
-; CHECK-NEXT:    sub v1.2s, v0.2s, v1.2s
-; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    movi v1.2s, #32
-; CHECK-NEXT:    clz v0.2s, v0.2s
-; CHECK-NEXT:    sub v0.2s, v1.2s, v0.2s
-; CHECK-NEXT:    ret
-    %b = call <2 x i32> @llvm.cttz.v2i32(<2 x i32> %a, i1 true)
-    ret <2 x i32> %b
-}
-
-define <1 x i64> @cttz_v1i64(<1 x i64> %a) nounwind {
-; CHECK-LABEL: cttz_v1i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    fmov d1, x8
-; CHECK-NEXT:    sub d1, d0, d1
-; CHECK-NEXT:    bic v0.8b, v1.8b, v0.8b
-; CHECK-NEXT:    cnt v0.8b, v0.8b
-; CHECK-NEXT:    uaddlp v0.4h, v0.8b
-; CHECK-NEXT:    uaddlp v0.2s, v0.4h
-; CHECK-NEXT:    uaddlp v0.1d, v0.2s
-; CHECK-NEXT:    ret
-    %b = call <1 x i64> @llvm.cttz.v1i64(<1 x i64> %a, i1 true)
-    ret <1 x i64> %b
-}
-
-define <16 x i8> @cttz_v16i8(<16 x i8> %a) nounwind {
-; CHECK-LABEL: cttz_v16i8:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.16b, #1
-; CHECK-NEXT:    sub v1.16b, v0.16b, v1.16b
-; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    ret
-    %b = call <16 x i8> @llvm.cttz.v16i8(<16 x i8> %a, i1 true)
-    ret <16 x i8> %b
-}
-
-define <8 x i16> @cttz_v8i16(<8 x i16> %a) nounwind {
-; CHECK-LABEL: cttz_v8i16:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.8h, #1
-; CHECK-NEXT:    sub v1.8h, v0.8h, v1.8h
-; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    movi v1.8h, #16
-; CHECK-NEXT:    clz v0.8h, v0.8h
-; CHECK-NEXT:    sub v0.8h, v1.8h, v0.8h
-; CHECK-NEXT:    ret
-    %b = call <8 x i16> @llvm.cttz.v8i16(<8 x i16> %a, i1 true)
-    ret <8 x i16> %b
-}
-
-define <4 x i32> @cttz_v4i32(<4 x i32> %a) nounwind {
-; CHECK-LABEL: cttz_v4i32:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    movi v1.4s, #1
-; CHECK-NEXT:    sub v1.4s, v0.4s, v1.4s
-; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    movi v1.4s, #32
-; CHECK-NEXT:    clz v0.4s, v0.4s
-; CHECK-NEXT:    sub v0.4s, v1.4s, v0.4s
-; CHECK-NEXT:    ret
-    %b = call <4 x i32> @llvm.cttz.v4i32(<4 x i32> %a, i1 true)
-    ret <4 x i32> %b
-}
-
-define <2 x i64> @cttz_v2i64(<2 x i64> %a) nounwind {
-; CHECK-LABEL: cttz_v2i64:
-; CHECK:       // %bb.0:
-; CHECK-NEXT:    mov w8, #1 // =0x1
-; CHECK-NEXT:    dup v1.2d, x8
-; CHECK-NEXT:    sub v1.2d, v0.2d, v1.2d
-; CHECK-NEXT:    bic v0.16b, v1.16b, v0.16b
-; CHECK-NEXT:    cnt v0.16b, v0.16b
-; CHECK-NEXT:    uaddlp v0.8h, v0.16b
-; CHECK-NEXT:    uaddlp v0.4s, v0.8h
-; CHECK-NEXT:    uaddlp v0.2d, v0.4s
-; CHECK-NEXT:    ret
-    %b = call <2 x i64> @llvm.cttz.v2i64(<2 x i64> %a, i1 true)
-    ret <2 x i64> %b
-}